Skip to content

public to internal for src/Core #160

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions src/CharsetDetector.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ namespace UtfUnknown
/// </summary>
public class CharsetDetector
{
internal InputState InputState;
private InputState _inputState;

/// <summary>
/// Start of the file
Expand Down Expand Up @@ -89,7 +89,7 @@ private IList<CharsetProber> CharsetProbers
{
get
{
switch (InputState)
switch (_inputState)
{
case InputState.EscASCII:
return _escCharsetProber;
Expand All @@ -112,7 +112,7 @@ private IList<CharsetProber> CharsetProbers
private CharsetDetector()
{
_start = true;
InputState = InputState.PureASCII;
_inputState = InputState.PureASCII;
_lastChar = 0x00;
}

Expand Down Expand Up @@ -350,9 +350,9 @@ private void FindInputState(byte[] buf, int offset, int len)
if ((buf[i] & 0x80) != 0 && buf[i] != 0xA0)
{
// we got a non-ascii byte (high-byte)
if (InputState != InputState.Highbyte)
if (_inputState != InputState.Highbyte)
{
InputState = InputState.Highbyte;
_inputState = InputState.Highbyte;

// kill EscCharsetProber if it is active
_escCharsetProber = null;
Expand All @@ -361,11 +361,11 @@ private void FindInputState(byte[] buf, int offset, int len)
}
else
{
if (InputState == InputState.PureASCII &&
if (_inputState == InputState.PureASCII &&
(buf[i] == 0x1B || (buf[i] == 0x7B && _lastChar == 0x7E)))
{
// found escape character or HZ "~{"
InputState = InputState.EscASCII;
_inputState = InputState.EscASCII;
_escCharsetProber = _escCharsetProber ?? GetNewProbers();
}
_lastChar = buf[i];
Expand Down Expand Up @@ -453,7 +453,7 @@ private DetectionResult DataEnd()
return new DetectionResult(_detectionDetail);
}

if (InputState == InputState.Highbyte)
if (_inputState == InputState.Highbyte)
{
var detectionResults = _charsetProbers
.Select(prober => new DetectionDetail(prober))
Expand All @@ -465,12 +465,12 @@ private DetectionResult DataEnd()

//TODO why done isn't true?
}
else if (InputState == InputState.PureASCII)
else if (_inputState == InputState.PureASCII)
{
//TODO why done isn't true?
return new DetectionResult(new DetectionDetail(CodepageName.ASCII, 1.0f));
}
else if (InputState == InputState.EscASCII)
else if (_inputState == InputState.EscASCII)
{
return new DetectionResult(new DetectionDetail(CodepageName.ASCII, 1.0f));
}
Expand All @@ -480,7 +480,7 @@ private DetectionResult DataEnd()

internal IList<CharsetProber> GetNewProbers()
{
switch (InputState)
switch (_inputState)
{
case InputState.EscASCII:
return new List<CharsetProber>() { new EscCharsetProber() };
Expand All @@ -499,4 +499,4 @@ internal IList<CharsetProber> GetNewProbers()
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Chinese
{
public class BIG5DistributionAnalyser : CharDistributionAnalyser
internal class BIG5DistributionAnalyser : CharDistributionAnalyser
{
// Big5 frequency table
// by Taiwan's Mandarin Promotion Council
Expand Down Expand Up @@ -926,4 +926,4 @@ public override int GetOrder(byte[] buf, int offset)
}
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Chinese
{
public class EUCTWDistributionAnalyser : CharDistributionAnalyser
internal class EUCTWDistributionAnalyser : CharDistributionAnalyser
{
// EUCTW frequency table
// Converted from big5 work
Expand Down Expand Up @@ -425,4 +425,4 @@ public override int GetOrder(byte[] buf, int offset)
return -1;
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Chinese
{
public class GB18030DistributionAnalyser : CharDistributionAnalyser
internal class GB18030DistributionAnalyser : CharDistributionAnalyser
{
// GB2312 most frequently used character table
// Char to FreqOrder table, from hz6763
Expand Down Expand Up @@ -471,4 +471,4 @@ public override int GetOrder(byte[] buf, int offset)
return -1;
}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Analyzers/MultiByte/Japanese/EUCJPContextAnalyser.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Japanese
{
public class EUCJPContextAnalyser : JapaneseContextAnalyser
internal class EUCJPContextAnalyser : JapaneseContextAnalyser
{
private const byte HIRAGANA_FIRST_BYTE = 0xA4;

Expand Down Expand Up @@ -36,4 +36,4 @@ protected override int GetOrder(byte[] buf, int offset)
return -1;
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Japanese
{
public class EUCJPDistributionAnalyser : SJISDistributionAnalyser
internal class EUCJPDistributionAnalyser : SJISDistributionAnalyser
{
/// <summary>
/// first byte range: 0xa0 -- 0xfe
Expand All @@ -15,4 +15,4 @@ public override int GetOrder(byte[] buf, int offset)
return -1;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@

namespace UtfUnknown.Core.Analyzers.Japanese
{
public abstract class JapaneseContextAnalyser
internal abstract class JapaneseContextAnalyser
{
protected const int CATEGORIES_NUM = 6;
protected const int ENOUGH_REL_THRESHOLD = 100;
Expand Down
4 changes: 2 additions & 2 deletions src/Core/Analyzers/MultiByte/Japanese/SJISContextAnalyser.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Japanese
{
public class SJISContextAnalyser : JapaneseContextAnalyser
internal class SJISContextAnalyser : JapaneseContextAnalyser
{
private const byte HIRAGANA_FIRST_BYTE = 0x82;

Expand Down Expand Up @@ -33,4 +33,4 @@ protected override int GetOrder(byte[] buf, int offset)
return -1;
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Japanese
{
public class SJISDistributionAnalyser : CharDistributionAnalyser
internal class SJISDistributionAnalyser : CharDistributionAnalyser
{
// Sampling from about 20M text materials include literature and computer technology
// Japanese frequency table, applied to both S-JIS and EUC-JP
Expand Down Expand Up @@ -575,4 +575,4 @@ public override int GetOrder(byte[] buf, int offset)
return order;
}
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Analyzers.Korean
{
public class EUCKRDistributionAnalyser : CharDistributionAnalyser
internal class EUCKRDistributionAnalyser : CharDistributionAnalyser
{
// Sampling from about 20M text materials include literature and computer technology

Expand Down
28 changes: 14 additions & 14 deletions src/Core/BitPackage.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,23 +38,23 @@

namespace UtfUnknown.Core
{
public class BitPackage
internal class BitPackage
{
public static int INDEX_SHIFT_4BITS = 3;
public static int INDEX_SHIFT_8BITS = 2;
public static int INDEX_SHIFT_16BITS = 1;
public static int SHIFT_MASK_4BITS = 7;
public static int SHIFT_MASK_8BITS = 3;
public static int INDEX_SHIFT_4BITS => 3;
public static int INDEX_SHIFT_8BITS => 2;
public static int INDEX_SHIFT_16BITS => 1;
public static int SHIFT_MASK_4BITS => 7;
public static int SHIFT_MASK_8BITS => 3;

public static int SHIFT_MASK_16BITS = 1;
public static int BIT_SHIFT_4BITS = 2;
public static int BIT_SHIFT_8BITS = 3;
public static int SHIFT_MASK_16BITS => 1;
public static int BIT_SHIFT_4BITS => 2;
public static int BIT_SHIFT_8BITS => 3;

public static int BIT_SHIFT_16BITS = 4;
public static int UNIT_MASK_4BITS = 0x0000000F;
public static int UNIT_MASK_8BITS = 0x000000FF;
public static int BIT_SHIFT_16BITS => 4;
public static int UNIT_MASK_4BITS => 0x0000000F;
public static int UNIT_MASK_8BITS => 0x000000FF;

public static int UNIT_MASK_16BITS = 0x0000FFFF;
public static int UNIT_MASK_16BITS => 0x0000FFFF;

private int indexShift;
private int shiftMask;
Expand Down Expand Up @@ -95,4 +95,4 @@ public int Unpack(int i)
((i & shiftMask) << bitShift)) & unitMask;
}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Chinese/BIG5SMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Chinese
{
public class BIG5SMModel : StateMachineModel
internal class BIG5SMModel : StateMachineModel
{
private readonly static int[] BIG5_cls = {
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07
Expand Down Expand Up @@ -59,4 +59,4 @@ public BIG5SMModel() : base(
{
}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Chinese/EUCTWSMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Chinese
{
public class EUCTWSMModel : StateMachineModel
internal class EUCTWSMModel : StateMachineModel
{
private readonly static int[] EUCTW_cls = {
BitPackage.Pack4bits(2,2,2,2,2,2,2,2), // 00 - 07
Expand Down Expand Up @@ -62,4 +62,4 @@ public EUCTWSMModel() : base(
{
}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Chinese/GB18030_SMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Chinese
{
public class GB18030_SMModel : StateMachineModel
internal class GB18030_SMModel : StateMachineModel
{
private readonly static int[] GB18030_cls = {
BitPackage.Pack4bits(1,1,1,1,1,1,1,1), // 00 - 07
Expand Down Expand Up @@ -67,4 +67,4 @@ public GB18030_SMModel() : base(
{
}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Chinese/HZ_GB_2312_SMModel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
//Escaped charsets state machines
namespace UtfUnknown.Core.Models.MultiByte.Chinese
{
public class HZ_GB_2312_SMModel : StateMachineModel
internal class HZ_GB_2312_SMModel : StateMachineModel
{
private readonly static int[] HZ_cls = {
BitPackage.Pack4bits(1,0,0,0,0,0,0,0), // 00 - 07
Expand Down Expand Up @@ -101,4 +101,4 @@ public HZ_GB_2312_SMModel() : base(
{
}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Chinese/Iso_2022_CN_SMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Chinese
{
public class Iso_2022_CN_SMModel : StateMachineModel
internal class Iso_2022_CN_SMModel : StateMachineModel
{
private readonly static int[] ISO2022CN_cls = {
BitPackage.Pack4bits(2,0,0,0,0,0,0,0), // 00 - 07
Expand Down Expand Up @@ -64,4 +64,4 @@ public Iso_2022_CN_SMModel() : base(
{
}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Japanese/EUCJPSMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Japanese
{
public class EUCJPSMModel : StateMachineModel
internal class EUCJPSMModel : StateMachineModel
{
private readonly static int[] EUCJP_cls = {
//BitPacket.Pack4bits(5,4,4,4,4,4,4,4), // 00 - 07
Expand Down Expand Up @@ -63,4 +63,4 @@ public EUCJPSMModel() : base(

}
}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Japanese/Iso_2022_JP_SMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Japanese
{
public class Iso_2022_JP_SMModel : StateMachineModel
internal class Iso_2022_JP_SMModel : StateMachineModel
{
private readonly static int[] ISO2022JP_cls = {
BitPackage.Pack4bits(2,0,0,0,0,0,0,0), // 00 - 07
Expand Down Expand Up @@ -67,4 +67,4 @@ public Iso_2022_JP_SMModel() : base(
}

}
}
}
4 changes: 2 additions & 2 deletions src/Core/Models/MultiByte/Japanese/SJIS_SMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Japanese
{
public class SJIS_SMModel : StateMachineModel
internal class SJIS_SMModel : StateMachineModel
{
private readonly static int[] SJIS_cls = {
//BitPacket.Pack4bits(0,1,1,1,1,1,1,1), // 00 - 07
Expand Down Expand Up @@ -63,4 +63,4 @@ public SJIS_SMModel() : base(

}
}
}
}
2 changes: 1 addition & 1 deletion src/Core/Models/MultiByte/Korean/CP949SMModel.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
namespace UtfUnknown.Core.Models.MultiByte.Korean
{
public class CP949SMModel : StateMachineModel
internal class CP949SMModel : StateMachineModel
{
/*
* 0: Unused
Expand Down
Loading