x/ctx
1
0
Fork 0
mirror of https://github.com/skidoodle/ctx.git synced 2026-04-28 03:07:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

Refine TokenCounter handling and output

Browse Source 
Decode across Write calls and buffer partial runes. Approximate
tokenization by splitting long alphanumeric runs (>4) and properly count
spaces and punctuation. Remove TokenCounter Printf/Println helpers and
update callers to use fmt.Fprintf/Fprintln. Avoid writing when the
underlying writer is nil.
This commit is contained in:
albert
2026-01-31 03:24:17 +01:00
parent 4453856fd1
commit 14dfacea29
2 changed files with 64 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
output.go
+14 -14
View File
@@ -30,19 +30,19 @@ func writeOutput(root string, files []string, outputPath string) (count int64, e
tc := &TokenCounter{w: bw}
tc.Printf("Project Path: %s\n\n", filepath.Base(root))
tc.Println("Source Tree:")
tc.Println("")
fmt.Fprintf(tc, "Project Path: %s\n\n", filepath.Base(root))
fmt.Fprintln(tc, "Source Tree:")
fmt.Fprintln(tc, "")
tc.Println("```txt")
tc.Println(filepath.Base(root))
fmt.Fprintln(tc, "```txt")
fmt.Fprintln(tc, filepath.Base(root))
if err := writeTree(tc, files); err != nil {
return 0, err
}
tc.Println("```")
tc.Println("")
fmt.Fprintln(tc, "```")
fmt.Fprintln(tc, "")
for _, file := range files {
if file == outputPath || filepath.Base(file) == outputPath {
@@ -52,7 +52,7 @@ func writeOutput(root string, files []string, outputPath string) (count int64, e
fullPath := filepath.Join(root, file)
content, err := os.ReadFile(fullPath)
if err != nil {
tc.Printf("Error reading %s: %v\n", file, err)
fmt.Fprintf(tc, "Error reading %s: %v\n", file, err)
continue
}
@@ -61,8 +61,8 @@ func writeOutput(root string, files []string, outputPath string) (count int64, e
ext = "txt"
}
tc.Printf("`%s`:\n\n", file)
tc.Printf("```%s\n", ext)
fmt.Fprintf(tc, "`%s`:\n\n", file)
fmt.Fprintf(tc, "```%s\n", ext)
if _, err := tc.Write(content); err != nil {
return 0, err
@@ -73,8 +73,8 @@ func writeOutput(root string, files []string, outputPath string) (count int64, e
return 0, err
}
}
tc.Println("```")
tc.Println("")
fmt.Fprintln(tc, "```")
fmt.Fprintln(tc, "")
}
return tc.Count, tc.Err
@@ -116,9 +116,9 @@ func printNode(w io.Writer, node map[string]any, prefix string) error {
children := node[key].(map[string]any)
if len(children) > 0 {
childPrefix := prefix + "│ "
childPrefix := prefix + "│   "
if isLast {
childPrefix = prefix + " "
childPrefix = prefix + "    "
}
if err := printNode(w, children, childPrefix); err != nil {
return err