Implement TASK-006A source content classifier

Pure-Kotlin classifier for feature/codebrowser/content/ that turns
already-loaded SourceContent bytes into bounded text, binary, Git LFS
pointer, unsupported-encoding, or oversized results with caller-supplied
positive byte/character/line limits, per handoff/task-006a-ready.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
최민영 2026-08-25 23:43:46 +09:00
parent 965d55d760
commit 4f5faf373a
5 changed files with 440 additions and 2 deletions

View File

@ -0,0 +1,24 @@
package com.ubisam.ubgitea.feature.codebrowser.content
sealed class ContentClassification {
abstract val sizeBytes: Int
data class Text(
override val sizeBytes: Int,
val renderedText: String,
val lineCount: Int,
val isTruncated: Boolean,
) : ContentClassification()
data class Binary(override val sizeBytes: Int) : ContentClassification()
data class LfsPointer(
override val sizeBytes: Int,
val objectId: String,
val objectSizeBytes: Long,
) : ContentClassification()
data class UnsupportedEncoding(override val sizeBytes: Int) : ContentClassification()
data class Oversized(override val sizeBytes: Int) : ContentClassification()
}

View File

@ -0,0 +1,105 @@
package com.ubisam.ubgitea.feature.codebrowser.content
import com.ubisam.ubgitea.domain.model.SourceContent
import java.nio.ByteBuffer
import java.nio.charset.CharacterCodingException
import java.nio.charset.CodingErrorAction
/**
* Classifies already-loaded repository bytes without ever guessing or silently
* substituting undecodable input. Byte, character, and line bounds are caller-supplied
* so the same classifier can be reused with different rendering budgets.
*/
class SourceContentClassifier(
private val maxInputBytes: Int,
private val maxRenderedCharacters: Int,
private val maxRenderedLines: Int,
) {
init {
require(maxInputBytes > 0) { "maxInputBytes must be positive" }
require(maxRenderedCharacters > 0) { "maxRenderedCharacters must be positive" }
require(maxRenderedLines > 0) { "maxRenderedLines must be positive" }
}
fun classify(content: SourceContent): ContentClassification {
val bytes = content.copyBytes()
val sizeBytes = bytes.size
if (sizeBytes > maxInputBytes) {
return ContentClassification.Oversized(sizeBytes)
}
if (bytes.any { it == NUL_BYTE }) {
return ContentClassification.Binary(sizeBytes)
}
val decoded = decodeStrictUtf8(bytes)
?: return ContentClassification.UnsupportedEncoding(sizeBytes)
parseLfsPointer(decoded, sizeBytes)?.let { return it }
return renderText(decoded, sizeBytes)
}
private fun decodeStrictUtf8(bytes: ByteArray): String? {
val decoder = Charsets.UTF_8.newDecoder()
.onMalformedInput(CodingErrorAction.REPORT)
.onUnmappableCharacter(CodingErrorAction.REPORT)
return try {
decoder.decode(ByteBuffer.wrap(bytes)).toString()
} catch (_: CharacterCodingException) {
null
}
}
// Git LFS pointer files are required to be UTF-8 text no larger than 1024 bytes,
// with `version` as the first line: https://github.com/git-lfs/git-lfs/blob/main/docs/spec.md
private fun parseLfsPointer(text: String, sizeBytes: Int): ContentClassification.LfsPointer? {
if (sizeBytes > LFS_POINTER_MAX_BYTES) return null
val lines = text.removeSuffix("\n").split("\n")
if (lines.size != 3 || lines[0] != LFS_VERSION_LINE) return null
val oidMatch = LFS_OID_LINE.matchEntire(lines[1]) ?: return null
val sizeMatch = LFS_SIZE_LINE.matchEntire(lines[2]) ?: return null
val objectSizeBytes = sizeMatch.groupValues[1].toLongOrNull() ?: return null
return ContentClassification.LfsPointer(
sizeBytes = sizeBytes,
objectId = oidMatch.groupValues[1],
objectSizeBytes = objectSizeBytes,
)
}
private fun renderText(text: String, sizeBytes: Int): ContentClassification.Text {
val lines = text.split("\n")
val lineCount = lines.size
val isLineTruncated = lineCount > maxRenderedLines
val boundedByLines = if (isLineTruncated) {
lines.take(maxRenderedLines).joinToString("\n")
} else {
text
}
val isCharTruncated = boundedByLines.length > maxRenderedCharacters
val rendered = if (isCharTruncated) {
boundedByLines.substring(0, maxRenderedCharacters)
} else {
boundedByLines
}
return ContentClassification.Text(
sizeBytes = sizeBytes,
renderedText = rendered,
lineCount = lineCount,
isTruncated = isLineTruncated || isCharTruncated,
)
}
private companion object {
const val NUL_BYTE: Byte = 0
const val LFS_POINTER_MAX_BYTES = 1024
const val LFS_VERSION_LINE = "version https://git-lfs.github.com/spec/v1"
val LFS_OID_LINE = Regex("^oid sha256:([0-9a-f]{64})$")
val LFS_SIZE_LINE = Regex("^size (0|[1-9][0-9]*)$")
}
}

View File

@ -0,0 +1,273 @@
package com.ubisam.ubgitea.feature.codebrowser.content
import com.ubisam.ubgitea.domain.model.SourceContent
import org.junit.Assert.assertEquals
import org.junit.Assert.assertFalse
import org.junit.Assert.assertTrue
import org.junit.Test
import kotlin.text.Charsets
class SourceContentClassifierTest {
private val classifier = SourceContentClassifier(
maxInputBytes = 1_000,
maxRenderedCharacters = 40,
maxRenderedLines = 5,
)
private fun content(text: String): SourceContent = SourceContent.from(text.toByteArray(Charsets.UTF_8))
// --- constructor limits ---
@Test(expected = IllegalArgumentException::class)
fun rejectsNonPositiveMaxInputBytes() {
SourceContentClassifier(maxInputBytes = 0, maxRenderedCharacters = 10, maxRenderedLines = 5)
}
@Test(expected = IllegalArgumentException::class)
fun rejectsNonPositiveMaxRenderedCharacters() {
SourceContentClassifier(maxInputBytes = 10, maxRenderedCharacters = 0, maxRenderedLines = 5)
}
@Test(expected = IllegalArgumentException::class)
fun rejectsNonPositiveMaxRenderedLines() {
SourceContentClassifier(maxInputBytes = 10, maxRenderedCharacters = 10, maxRenderedLines = 0)
}
// --- text ---
@Test
fun classifiesEmptyContentAsSingleEmptyLine() {
val result = classifier.classify(content(""))
assertEquals(ContentClassification.Text(sizeBytes = 0, renderedText = "", lineCount = 1, isTruncated = false), result)
}
@Test
fun classifiesAsciiTextExactly() {
val result = classifier.classify(content("fun main() {}"))
assertEquals(
ContentClassification.Text(sizeBytes = 13, renderedText = "fun main() {}", lineCount = 1, isTruncated = false),
result,
)
}
@Test
fun classifiesKoreanTextByCharacterNotByte() {
val text = "한글 텍스트"
val result = classifier.classify(content(text))
val expectedBytes = text.toByteArray(Charsets.UTF_8).size
assertEquals(
ContentClassification.Text(sizeBytes = expectedBytes, renderedText = text, lineCount = 1, isTruncated = false),
result,
)
}
@Test
fun preservesCarriageReturnsWithinLines() {
val result = classifier.classify(content("first\r\nsecond\r\n"))
val text = result as ContentClassification.Text
assertEquals(3, text.lineCount)
assertEquals("first\r\nsecond\r\n", text.renderedText)
assertFalse(text.isTruncated)
}
// --- binary ---
@Test
fun classifiesNulByteContentAsBinary() {
val bytes = byteArrayOf(0x50, 0x4b, 0x00, 0x03)
val result = classifier.classify(SourceContent.from(bytes))
assertEquals(ContentClassification.Binary(sizeBytes = 4), result)
}
// --- unsupported encoding ---
@Test
fun classifiesMalformedUtf8AsUnsupportedEncoding() {
val bytes = byteArrayOf(0x41, 0xFF.toByte(), 0x42)
val result = classifier.classify(SourceContent.from(bytes))
assertEquals(ContentClassification.UnsupportedEncoding(sizeBytes = 3), result)
}
@Test
fun classifiesTruncatedMultiByteSequenceAsUnsupportedEncoding() {
// 0xE2 0x82 is the first two bytes of a three-byte sequence, cut short.
val bytes = byteArrayOf(0x41, 0xE2.toByte(), 0x82.toByte())
val result = classifier.classify(SourceContent.from(bytes))
assertEquals(ContentClassification.UnsupportedEncoding(sizeBytes = 3), result)
}
// --- Git LFS pointer ---
@Test
fun classifiesCanonicalLfsPointer() {
val pointer = "version https://git-lfs.github.com/spec/v1\n" +
"oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n" +
"size 12345\n"
val result = classifier.classify(content(pointer))
assertEquals(
ContentClassification.LfsPointer(
sizeBytes = pointer.toByteArray(Charsets.UTF_8).size,
objectId = "4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393",
objectSizeBytes = 12345L,
),
result,
)
}
@Test
fun classifiesLfsPointerWithoutTrailingNewline() {
val pointer = "version https://git-lfs.github.com/spec/v1\n" +
"oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n" +
"size 0"
val result = classifier.classify(content(pointer))
assertTrue(result is ContentClassification.LfsPointer)
assertEquals(0L, (result as ContentClassification.LfsPointer).objectSizeBytes)
}
@Test
fun rejectsLfsPointerWithWrongVersionLine() {
val pointer = "version https://git-lfs.github.com/spec/v0\n" +
"oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n" +
"size 12345\n"
val result = classifier.classify(content(pointer))
assertTrue(result is ContentClassification.Text)
}
@Test
fun rejectsLfsPointerWithMalformedOid() {
val pointer = "version https://git-lfs.github.com/spec/v1\n" +
"oid sha256:not-a-valid-hash\n" +
"size 12345\n"
val result = classifier.classify(content(pointer))
assertTrue(result is ContentClassification.Text)
}
@Test
fun rejectsLfsPointerWithNonNumericSize() {
val pointer = "version https://git-lfs.github.com/spec/v1\n" +
"oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n" +
"size abc\n"
val result = classifier.classify(content(pointer))
assertTrue(result is ContentClassification.Text)
}
@Test
fun rejectsLfsPointerWithMissingRequiredLine() {
val pointer = "version https://git-lfs.github.com/spec/v1\n" +
"oid sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393\n"
val result = classifier.classify(content(pointer))
assertTrue(result is ContentClassification.Text)
}
// --- byte limits ---
@Test
fun classifiesInputAtExactByteLimitAsText() {
val exact = SourceContentClassifier(maxInputBytes = 5, maxRenderedCharacters = 40, maxRenderedLines = 5)
val result = exact.classify(content("abcde"))
assertTrue(result is ContentClassification.Text)
}
@Test
fun classifiesInputOverByteLimitAsOversizedWithoutContent() {
val exact = SourceContentClassifier(maxInputBytes = 5, maxRenderedCharacters = 40, maxRenderedLines = 5)
val result = exact.classify(content("abcdef"))
assertEquals(ContentClassification.Oversized(sizeBytes = 6), result)
}
@Test
fun oversizedBinaryContentIsRejectedBeforeBinaryDetection() {
val exact = SourceContentClassifier(maxInputBytes = 2, maxRenderedCharacters = 40, maxRenderedLines = 5)
val bytes = byteArrayOf(0x00, 0x00, 0x00)
val result = exact.classify(SourceContent.from(bytes))
assertEquals(ContentClassification.Oversized(sizeBytes = 3), result)
}
// --- line limits ---
@Test
fun classifiesInputAtExactLineLimitAsNotTruncated() {
val limited = SourceContentClassifier(maxInputBytes = 1_000, maxRenderedCharacters = 1_000, maxRenderedLines = 3)
val result = limited.classify(content("a\nb\nc"))
val text = result as ContentClassification.Text
assertEquals(3, text.lineCount)
assertFalse(text.isTruncated)
assertEquals("a\nb\nc", text.renderedText)
}
@Test
fun classifiesInputOverLineLimitAsTruncated() {
val limited = SourceContentClassifier(maxInputBytes = 1_000, maxRenderedCharacters = 1_000, maxRenderedLines = 3)
val result = limited.classify(content("a\nb\nc\nd"))
val text = result as ContentClassification.Text
assertEquals(4, text.lineCount)
assertTrue(text.isTruncated)
assertEquals("a\nb\nc", text.renderedText)
}
// --- character limits ---
@Test
fun classifiesInputAtExactCharacterLimitAsNotTruncated() {
val limited = SourceContentClassifier(maxInputBytes = 1_000, maxRenderedCharacters = 5, maxRenderedLines = 5)
val result = limited.classify(content("abcde"))
val text = result as ContentClassification.Text
assertFalse(text.isTruncated)
assertEquals("abcde", text.renderedText)
}
@Test
fun classifiesInputOverCharacterLimitAsTruncated() {
val limited = SourceContentClassifier(maxInputBytes = 1_000, maxRenderedCharacters = 5, maxRenderedLines = 5)
val result = limited.classify(content("abcdef"))
val text = result as ContentClassification.Text
assertTrue(text.isTruncated)
assertEquals("abcde", text.renderedText)
}
// --- defensive immutability ---
@Test
fun classificationIsUnaffectedByMutatingSourceBytesAfterConstruction() {
val bytes = "original".toByteArray(Charsets.UTF_8)
val source = SourceContent.from(bytes)
bytes[0] = 'X'.code.toByte()
val result = classifier.classify(source)
assertEquals(
ContentClassification.Text(sizeBytes = 8, renderedText = "original", lineCount = 1, isTruncated = false),
result,
)
}
@Test
fun repeatedClassificationOfSameContentIsStable() {
val source = content("stable text")
val first = classifier.classify(source)
val second = classifier.classify(source)
assertEquals(first, second)
}
}

View File

@ -86,3 +86,16 @@ This is the append-only implementation and handoff record shared by Codex and Cl
- Review result: No handoff activation occurs until the tag resolves to the committed metadata state. Claude checks out the tag, creates a `claude/` branch, implements TASK-006A only, and then stops for Codex review.
- Known limitation: Repository mailboxes are asynchronous and must be checked manually at task start.
- Next owner: Claude for TASK-006A.
## 2026-08-25 — TASK-006A implementation
- Agent and role: Claude, assigned feature implementer (writable scope limited to `feature/codebrowser/content/` and this task's evidence/log entries)
- Commit/diff: branch `claude/task-006a-ready` created from activation tag `handoff/task-006a-ready` (base `965d55d`); implementation adds `ContentClassification.kt`, `SourceContentClassifier.kt`, and `SourceContentClassifierTest.kt` under `feature/codebrowser/content/`
- Features implemented: Pure-Kotlin classifier that turns already-loaded `SourceContent` bytes into one of five immutable states — bounded text (with independent line/character truncation flags), binary, Git LFS pointer, unsupported encoding, or oversized — using caller-supplied positive byte/character/line limits.
- How it was implemented: Oversized input is rejected by raw byte count before any decoding; binary is detected by a raw NUL-byte scan; text decoding uses a strict UTF-8 `CharsetDecoder` with `CodingErrorAction.REPORT` for malformed and unmappable input so it never substitutes replacement characters; a Git LFS pointer is recognized only from the exact canonical `version` line followed by valid `oid sha256:<64 hex>` and `size <integer>` lines within the 1024-byte spec bound; remaining text is truncated independently by line count and character count with an explicit `isTruncated` flag. No shared domain, data, Gradle, manifest, or navigation file was modified; `RepositoryModels.kt`/`RepositoryPath.kt`/`RepositoryRef.kt` remain identical to contract-freeze commit `0929282`.
- Files changed: `app/src/main/java/com/ubisam/ubgitea/feature/codebrowser/content/ContentClassification.kt`, `app/src/main/java/com/ubisam/ubgitea/feature/codebrowser/content/SourceContentClassifier.kt`, `app/src/test/java/com/ubisam/ubgitea/feature/codebrowser/content/SourceContentClassifierTest.kt`, `tasks/006a-code-content-classifier.md` (evidence/status), this log entry.
- Checks and results: `./gradlew testDebugUnitTest --tests "com.ubisam.ubgitea.feature.codebrowser.content.*"` passed (25/25); `./gradlew assembleDebug testDebugUnitTest lintDebug --rerun-tasks` passed (82 unit tests repository-wide, 0 failures; lint 0 errors, 22 pre-existing warnings unchanged from TASK-001 baseline); `git diff --check` clean; sensitive-pattern scan of changed files found no secret/token/password/private-key markers.
- Review findings and resolutions: None yet — pending Codex read-only review.
- Known limitations: No Android device/emulator was used, since this task has no Android framework dependency (pure JVM-testable Kotlin); static formatting/check task remains unconfigured for the module.
- Next recommended task: Codex reviews this commit read-only per `docs/workstreams.md` cross-review rules; TASK-006 UI remains blocked until Codex freezes the TASK-004/005 contracts.
- Next owner: Codex, for review.

View File

@ -1,6 +1,6 @@
# TASK-006A: Classify source content safely
Status: ready — start from immutable Git tag `handoff/task-006a-ready`
Status: implemented by Claude; pending Codex read-only review
Owner: Claude
Reviewer: Codex
@ -55,4 +55,27 @@ Claude must provide the exact branch/commit, changed paths, acceptance-criteria
## Evidence
Pending Claude implementation.
Implemented on branch `claude/task-006a-code-content-classifier` from activation tag `handoff/task-006a-ready` (base commit `965d55d`), using JDK 21 (Android Studio bundled JBR at `D:\AndroidStudio\jbr`) and Android SDK 36/Build-Tools 35.
- Files added (writable scope only): `app/src/main/java/com/ubisam/ubgitea/feature/codebrowser/content/ContentClassification.kt`, `app/src/main/java/com/ubisam/ubgitea/feature/codebrowser/content/SourceContentClassifier.kt`, `app/src/test/java/com/ubisam/ubgitea/feature/codebrowser/content/SourceContentClassifierTest.kt`.
- No shared domain, data, Gradle, manifest, navigation, or generic resource file was touched. `RepositoryModels.kt`, `RepositoryPath.kt`, and `RepositoryRef.kt` under `domain/model/` remain byte-for-byte identical to contract-freeze commit `0929282`.
- Design: `ContentClassification` is a sealed class of five data classes (`Text`, `Binary`, `LfsPointer`, `UnsupportedEncoding`, `Oversized`), each holding only primitive/`String`/`Long` fields, so no defensive copy is needed for immutability. `SourceContentClassifier` takes three positive constructor-injected limits (`maxInputBytes`, `maxRenderedCharacters`, `maxRenderedLines`) and classifies in order: oversized (raw byte count vs. limit, before any decode) → binary (raw NUL-byte scan) → strict UTF-8 decode (`CodingErrorAction.REPORT` for both malformed and unmappable input, so failures return `UnsupportedEncoding` instead of substituting replacement characters) → Git LFS pointer (exact canonical `version` line first, then `oid sha256:<64 hex>` and `size <non-negative integer>` lines, bounded to 1024 bytes per the upstream git-lfs spec) → bounded text with independent line-count and character-count truncation flags.
- Acceptance-criteria checklist: all nine boxes above are satisfied — classification set is immutable and covers all five states; limits are constructor-injected and validated positive (`init` block); oversized content is rejected before decoding and exposes only `sizeBytes`; binary detection uses a raw NUL scan ahead of decoding, and decoding never substitutes replacement characters; UTF-8 decoding is strict; LFS recognition requires the exact version line plus valid oid/size fields within the 1024-byte bound; `Text.isTruncated` marks both line- and character-limit overflow; the package imports only `com.ubisam.ubgitea.domain.model` and `java.nio.*`/Kotlin stdlib, no Android/network/persistence/token/logging/write dependency.
- Test coverage (25 cases in `SourceContentClassifierTest`): constructor limit validation (3), empty/ASCII/Korean text (3), CRLF preservation (1), NUL/binary (1), malformed and truncated-sequence UTF-8 (2), canonical and no-trailing-newline LFS pointers (2), malformed LFS pointers — wrong version, bad oid, non-numeric size, missing line (4), exact/over byte limit (2), oversized-and-binary precedence (1), exact/over line limit (2), exact/over character limit (2), defensive immutability against post-construction byte mutation and repeated classification (2). Only synthetic fixtures are used; no error path includes decoded source content in its result.
Checks run from the new worktree (`local.properties` created locally for the SDK path, not committed):
```text
./gradlew testDebugUnitTest --tests "com.ubisam.ubgitea.feature.codebrowser.content.*" → BUILD SUCCESSFUL, 25/25 passed
./gradlew assembleDebug testDebugUnitTest lintDebug --rerun-tasks → BUILD SUCCESSFUL
- testDebugUnitTest: 82 tests total (repository-wide), 0 failures, 0 errors
- lintDebug: 0 errors, 22 pre-existing warnings (unrelated to this change; same count as TASK-001 baseline)
git diff --check → no whitespace errors
sensitive-pattern scan of changed files (secret/token/password/private-key/API-key markers) → no matches
```
Known limitations:
- No Android device/emulator was available; this task is pure JVM unit-testable Kotlin with no Android framework dependency, so no manual device step applies.
- Static formatting/check task is not configured for this module (unchanged from TASK-001).
No open P0-P3 findings from self-review against the acceptance criteria. Requesting Codex read-only review per `docs/workstreams.md` cross-review rules; Claude stops writing after this handoff.