Fixed decoding UTF-8 encoded string literals

Author: Atvaark

.gitignore

@@ -5,6 +5,7 @@
 *.suo
 *.user
 *.sln.docstates
+/.vs/
 
 # Build results
 [Dd]ebug/

FoxTool/Constants.cs

@@ -0,0 +1,9 @@
+using System.Text;
+
+namespace FoxTool
+{
+    internal static class Constants
+    {
+        public static readonly Encoding StringEncoding = Encoding.UTF8;
+    }
+}

FoxTool/ExtensionMethods.cs

@@ -37,9 +37,10 @@ internal static void AlignWrite(this Stream output, int alignment, byte data)
             }
         }
 
-        internal static string ReadString(this BinaryReader binaryReader, int count)
+        internal static string ReadString(this BinaryReader binaryReader, int byteCount)
         {
-            return new string(binaryReader.ReadChars(count));
+            byte[] bytes = binaryReader.ReadBytes(byteCount);
+            return Constants.StringEncoding.GetString(bytes);
         }
 
         internal static bool IsPrintable(this string s)

FoxTool/Fox/FoxFile.cs

@@ -207,7 +207,7 @@ private void Read(Stream input)
             }
             input.AlignRead(16);
             reader.Skip(2);
-            string eof = reader.ReadString(3);
+            byte[] eof = reader.ReadBytes(3);
             input.AlignRead(16);
         }
 
@@ -229,7 +229,9 @@ private Dictionary<ulong, string> GenerateLocalLookupTable()
                 if (localLookupTable.ContainsKey(literal.Hash.HashValue) == false)
                 {
                     if (literal.IsEncrypted == false)
+                    {
                         localLookupTable.Add(literal.Hash.HashValue, literal.Literal);
+                    }
                 }
             }
             return localLookupTable;

FoxTool/Fox/FoxLookupTable.cs

@@ -27,9 +27,15 @@ public string Lookup(ulong hash)
         {
             string result;
             if (LocalLookupTable.TryGetValue(hash, out result))
+            {
                 return result;
+            }
+
             if (GlobalLookupTable.TryGetValue(hash, out result))
+            {
                 return result;
+            }
+
             return null;
         }
     }

FoxTool/Fox/FoxStringLiteralBase.cs

@@ -61,9 +61,10 @@ public override int GetHashCode()
 
         public void CheckForEncryption()
         {
-            if (Hashing.HashString(Literal) != Hash.HashValue)
+            ulong literalHash = Hashing.HashString(Literal);
+            if (literalHash != Hash.HashValue)
             {
-                EncryptedLiteral = Encoding.Default.GetBytes(Literal);
+                EncryptedLiteral = Constants.StringEncoding.GetBytes(Literal);
             }
         }
 

FoxTool/Fox/FoxStringLookupLiteral.cs

@@ -42,7 +42,7 @@ public override void Write(Stream output)
         {
             BinaryWriter writer = new BinaryWriter(output, Encoding.Default, true);
 
-            byte[] nameBytes = Literal == null ? new byte[0] : Encoding.Default.GetBytes(Literal);
+            byte[] nameBytes = Literal == null ? new byte[0] : Constants.StringEncoding.GetBytes(Literal);
 
             Hash.Write(output);
             writer.Write((uint) nameBytes.Length);

FoxTool/FoxConverter.cs

@@ -12,7 +12,7 @@ public static void DecompileFox(FoxFile foxFile, Stream output)
         {
             var settings = new XmlWriterSettings
             {
-                Encoding = Encoding.UTF8,
+                Encoding = Constants.StringEncoding,
                 Indent = true
             };
 

FoxTool/FoxTool.csproj

@@ -34,9 +34,8 @@
     <WarningLevel>4</WarningLevel>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="CityHash, Version=0.1.1.0, Culture=neutral, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\CityHash.Net.Legacy.0.1.1\lib\net45\CityHash.dll</HintPath>
+    <Reference Include="CityHash, Version=0.1.2.0, Culture=neutral, processorArchitecture=MSIL">
+      <HintPath>..\packages\CityHash.Net.Legacy.0.1.2.0\lib\net45\CityHash.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
@@ -56,6 +55,7 @@
     <Compile Include="Fox\Types\Structs\FoxPropertyInfo.cs" />
     <Compile Include="Fox\Types\Structs\FoxWideVector3.cs" />
     <Compile Include="Fox\Types\Values\FoxStringBase.cs" />
+    <Compile Include="Constants.cs" />
     <Compile Include="Tpp\Enums\TppLensFlareFieldInterpType.cs" />
     <Compile Include="Tpp\Enums\TppLensFlareFieldShapeType.cs" />
     <Compile Include="Tpp\Enums\TppLensFlareShapeDistanceScalingMode.cs" />

FoxTool/Hashing.cs

@@ -7,9 +7,11 @@ internal static class Hashing
         internal static ulong HashString(string text)
         {
             if (text == null) throw new ArgumentNullException("text");
+            byte[] bytes = Constants.StringEncoding.GetBytes(text + "\0");
             const ulong seed0 = 0x9ae16a3b2f90404f;
-            ulong seed1 = text.Length > 0 ? (uint) ((text[0]) << 16) + (uint) text.Length : 0;
-            return CityHash.CityHash.CityHash64WithSeeds(text + "\0", seed0, seed1) & 0xFFFFFFFFFFFF;
+            ulong seed1 = bytes.Length > 0 ? (uint) ((bytes[0]) << 16) + (uint) (bytes.Length - 1) : 0;
+            ulong hash = CityHash.CityHash.CityHash64WithSeeds(bytes, seed0, seed1) & 0xFFFFFFFFFFFF;
+            return hash;
         }
     }
 }

FoxTool/Properties/AssemblyInfo.cs

@@ -11,5 +11,5 @@
 [assembly: AssemblyCulture("")]
 [assembly: ComVisible(false)]
 [assembly: Guid("561aab12-c53d-4a0e-bac5-0efc04a54618")]
-[assembly: AssemblyVersion("0.2.4.0")]
-[assembly: AssemblyFileVersion("0.2.4.0")]
+[assembly: AssemblyVersion("0.2.5.0")]
+[assembly: AssemblyFileVersion("0.2.5.0")]

FoxTool/packages.config

@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="CityHash.Net.Legacy" version="0.1.1" targetFramework="net45" />
+  <package id="CityHash.Net.Legacy" version="0.1.2.0" targetFramework="net45" />
 </packages>