Use TokenCounter and correct unsafe pointer usage

Record write errors in TokenCounter.Printf and add Println method
to propagate write failures. Route file-read warnings to stderr
instead of mixing them into token output. Use unsafe.Add with a
base pointer when computing the drop target on Windows.

.github/workflows/release.yaml

@@ -22,6 +22,10 @@ jobs:
         with:
           go-version: stable
 
+      - name: Quick Sanity Check
+        shell: bash
+        run: go test -v ./...
+
       - name: Run GoReleaser
         uses: goreleaser/goreleaser-action@v6
         with:

.github/workflows/test.yaml

@@ -43,10 +43,11 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: |
           ./ctx.exe .
-          $files = Get-Clipboard -Format FileDropList
+          Add-Type -AssemblyName System.Windows.Forms
-          if ($files) {
+          $files = [System.Windows.Forms.Clipboard]::GetFileDropList()
-            Write-Host "Clipboard contains: $files"
+          if ($files.Contains("$PWD\ctx.txt")) {
-            if ($files.Name -contains "ctx.txt") { exit 0 }
+            Write-Host "Success: Clipboard contains ctx.txt"
+            exit 0
           }
-          Write-Error "Clipboard did not contain ctx.txt"
+          Write-Error "Clipboard did not contain ctx.txt. Found: $files"
           exit 1

clipboard/darwin.go

@@ -13,10 +13,13 @@ int copyFileToPasteboard(char* path) {
         NSString *strPath = [NSString stringWithUTF8String:path];
         if (!strPath) return 0;
 
+        NSURL *url = [NSURL fileURLWithPath:strPath];
+        if (!url) return 0;
+
         NSPasteboard *pb = [NSPasteboard generalPasteboard];
         [pb clearContents];
-        [pb declareTypes:@[NSFilenamesPboardType] owner:nil];
+
-        return [pb setPropertyList:@[strPath] forType:NSFilenamesPboardType] ? 1 : 0;
+        return [pb writeObjects:@[url]] ? 1 : 0;
     }
 }
 */

clipboard/windows.go

@@ -62,11 +62,13 @@ func CopyFile(path string) error {
 		return fmt.Errorf("GlobalLock failed")
 	}
 
-	df := (*dropFiles)(unsafe.Pointer(ptrVal))
+	basePtr := unsafe.Pointer(ptrVal)
+
+	df := (*dropFiles)(basePtr)
 	df.pFiles = dropSize
 	df.fWide = 1
 
-	targetPtr := unsafe.Pointer(ptrVal + uintptr(dropSize))
+	targetPtr := unsafe.Add(basePtr, dropSize)
 
 	srcSlice := unsafe.Slice((*uint16)(unsafe.Pointer(&pathUTF16[0])), len(pathUTF16))
 	dstSlice := unsafe.Slice((*uint16)(targetPtr), len(pathUTF16))

output.go

@@ -52,7 +52,7 @@ func writeOutput(root string, files []string, outputPath string) (count int64, e
 		fullPath := filepath.Join(root, file)
 		content, err := os.ReadFile(fullPath)
 		if err != nil {
-			tc.Printf("Error reading %s: %v\n", file, err)
+			fmt.Fprintf(os.Stderr, "ctx: warning: skipping %s: %v\n", file, err)
 			continue
 		}
 
@@ -116,9 +116,9 @@ func printNode(w io.Writer, node map[string]any, prefix string) error {
 
 		children := node[key].(map[string]any)
 		if len(children) > 0 {
-			childPrefix := prefix + "│   "
+			childPrefix := prefix + "│   "
 			if isLast {
-				childPrefix = prefix + "    "
+				childPrefix = prefix + "    "
 			}
 			if err := printNode(w, children, childPrefix); err != nil {
 				return err

token.go

@@ -4,14 +4,17 @@ import (
 	"fmt"
 	"io"
 	"unicode"
+	"unicode/utf8"
 )
 
 type TokenCounter struct {
-	w       io.Writer
+	w     io.Writer
-	Count   int64
+	Count int64
-	Err     error
+	Err   error
-	inWord  bool
+
-	inSpace bool
+	leftover []byte
+	inWord   bool
+	wordLen  int
 }
 
 func (tc *TokenCounter) Write(p []byte) (int, error) {
@@ -19,8 +22,29 @@ func (tc *TokenCounter) Write(p []byte) (int, error) {
 		return 0, tc.Err
 	}
 
-	for _, b := range p {
+	data := p
-		r := rune(b)
+	if len(tc.leftover) > 0 {
+		data = make([]byte, len(tc.leftover)+len(p))
+		copy(data, tc.leftover)
+		copy(data[len(tc.leftover):], p)
+	}
+
+	totalProcessed := 0
+
+	for len(data) > 0 {
+		r, size := utf8.DecodeRune(data)
+
+		if r == utf8.RuneError && size == 1 {
+			if len(data) < utf8.UTFMax {
+				tc.leftover = data
+				break
+			}
+		}
+
+		data = data[size:]
+		totalProcessed += size
+		tc.leftover = nil
+
 		isSpace := unicode.IsSpace(r)
 		isAlpha := unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_'
 
@@ -28,24 +52,32 @@ func (tc *TokenCounter) Write(p []byte) (int, error) {
 			if !tc.inWord {
 				tc.Count++
 				tc.inWord = true
-				tc.inSpace = false
+				tc.wordLen = 1
+			} else {
+				tc.wordLen++
+				if tc.wordLen > 4 {
+					tc.Count++
+					tc.wordLen = 1
+				}
 			}
 		} else if isSpace {
-			if !tc.inSpace {
+			tc.inWord = false
-				tc.Count++
+			tc.wordLen = 0
-				tc.inSpace = true
-				tc.inWord = false
-			}
 		} else {
 			tc.Count++
 			tc.inWord = false
-			tc.inSpace = false
+			tc.wordLen = 0
 		}
 	}
 
-	n, err := tc.w.Write(p)
+	var n int
-	tc.Err = err
+	if tc.w != nil {
-	return n, err
+		n, tc.Err = tc.w.Write(p)
+	} else {
+		n = len(p)
+	}
+
+	return n, tc.Err
 }
 
 func (tc *TokenCounter) WriteByte(c byte) error {
@@ -57,12 +89,12 @@ func (tc *TokenCounter) Printf(format string, a ...any) {
 	if tc.Err != nil {
 		return
 	}
-	_, _ = fmt.Fprintf(tc, format, a...)
+	_, tc.Err = fmt.Fprintf(tc, format, a...)
 }
 
 func (tc *TokenCounter) Println(a ...any) {
 	if tc.Err != nil {
 		return
 	}
-	_, _ = fmt.Fprintln(tc, a...)
+	_, tc.Err = fmt.Fprintln(tc, a...)
 }