recognize null terminated strings

rec0de · Dec 10, 2024 · bb09d0c · bb09d0c
1 parent dfd7a65
commit bb09d0c
Showing 1 changed file with 15 additions and 4 deletions.
diff --git a/src/commonMain/kotlin/decoders/BasicDecoders.kt b/src/commonMain/kotlin/decoders/BasicDecoders.kt
@@ -14,8 +14,10 @@ object Utf8Decoder : ByteWitchDecoder {
     override fun decodesAsValid(data: ByteArray) = confidence(data) > 0.6
 
     override fun confidence(data: ByteArray): Double {
+        val effectiveData = stripNullTerminator(data)
+
         try {
-            val score = looksLikeUtf8String(data)
+            val score = looksLikeUtf8String(effectiveData)
             //Logger.log(data.decodeToString())
             //Logger.log(score)
             return score
@@ -25,7 +27,7 @@ object Utf8Decoder : ByteWitchDecoder {
     }
 
     override fun decode(data: ByteArray, sourceOffset: Int, inlineDisplay: Boolean): ByteWitchResult {
-        return BWString(data.decodeToString(), Pair(sourceOffset, sourceOffset+data.size))
+        return BWString(stripNullTerminator(data).decodeToString(), Pair(sourceOffset, sourceOffset+data.size))
     }
 
     override fun tryhardDecode(data: ByteArray): ByteWitchResult? {
@@ -34,14 +36,23 @@ object Utf8Decoder : ByteWitchDecoder {
         else
             null
     }
+
+    fun stripNullTerminator(data: ByteArray): ByteArray {
+        // to hell with it, we'll support arbitrarily long null terminators
+        var end = data.size
+        while (end > 0 && data[end-1] == 0.toByte()) {
+            end -= 1
+        }
+        return data.sliceArray(0 until end)
+    }
 }
 
 object Utf16Decoder : ByteWitchDecoder {
     override val name = "utf16"
 
     override fun confidence(data: ByteArray): Double {
         try {
-            val string = data.decodeAsUTF16BE()
+            val string = Utf8Decoder.stripNullTerminator(data).decodeAsUTF16BE()
             return looksLikeUtf8String(string.encodeToByteArray())
         } catch (e: Exception) {
             return 0.0
@@ -51,7 +62,7 @@ object Utf16Decoder : ByteWitchDecoder {
     override fun decodesAsValid(data: ByteArray) = confidence(data) > 0.6
 
     override fun decode(data: ByteArray, sourceOffset: Int, inlineDisplay: Boolean): ByteWitchResult {
-        return BWString(data.decodeAsUTF16BE(), Pair(sourceOffset, sourceOffset+data.size))
+        return BWString(Utf8Decoder.stripNullTerminator(data).decodeAsUTF16BE(), Pair(sourceOffset, sourceOffset+data.size))
     }
 
     override fun tryhardDecode(data: ByteArray): ByteWitchResult? {