mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-15 06:16:22 +00:00
refactor: comments state current behavior, not project history
Comments across the tree justified the code by pointing at a Rust implementation that is no longer in the repository, citing files and line numbers (config.rs:132, schema.rs:26-54, reader.rs:42) that cannot be opened, plus crates and datasets that are equally gone. A reader could not check any of it. Every invariant those comments carried is kept and restated so it stands on its own: the bytewise sort that decides which row survives a duplicate exam number, the literal U+0300..U+036F range that must match the site's toAscii, the trailing space in "SINH ", the BIFF and shared-string corrections, the VACUUM-after-COMMIT rule, the deploy-from-main guard. The reader's contract is now anchored to the frozen oracle in parser/testdata, which still exists and is still checked, rather than to the tool that originally produced it. TestDDLMatchesRust becomes TestDDLIsFrozen: it compares against a copy of the DDL inside the test and never read schema.rs, so both the name and the failure message were misleading. ToAscii no longer claims the d-replacement must precede lowercasing. Both cases map to 'd' and ToLower runs last, so the order has no effect.
This commit is contained in:
1 parent
c988bfafcf
commit
933adf70c9
43 files changed
+392
-515
No files matched your search
@@ -1,9 +1,5 @@
|
||||
// Package transform performs row transformation: ASCII normalisation, score
|
||||
// regex parsing, and validation — a port of parser/src/transform.rs.
|
||||
//
|
||||
// ToAscii replicates build-lib.js toAscii exactly:
|
||||
//
|
||||
// str.normalize("NFD").replace(/[̀-ͯ]/g,"").replace(/đ/gi,"d").toLowerCase()
|
||||
// regex parsing, and validation.
|
||||
package transform
|
||||
|
||||
import (
|
||||
@@ -26,14 +22,11 @@ import (
|
||||
// 3. Replace đ/Đ with d (NFD does not decompose them)
|
||||
// 4. Lowercase
|
||||
//
|
||||
// Step 2 filters a LITERAL CODEPOINT RANGE, not a Unicode category. The inline
|
||||
// comment at transform.rs:53 says "Unicode category M", but the code at :56 is
|
||||
// the specification and it checks '\u{0300}'..='\u{036f}'. unicode.Is(unicode.Mn, r)
|
||||
// is strictly broader and would strip marks Rust keeps, silently changing
|
||||
// ho_ten_ascii — the column the site's accent-insensitive search runs on.
|
||||
//
|
||||
// Step order matters and mirrors transform.rs:54-63: the đ/Đ replacement happens
|
||||
// before lowercasing.
|
||||
// Step 2 filters a LITERAL CODEPOINT RANGE, not a Unicode category.
|
||||
// unicode.Is(unicode.Mn, r) is strictly broader and would strip marks this keeps,
|
||||
// silently changing ho_ten_ascii — the column the site's accent-insensitive
|
||||
// search runs on. That search normalises the query with the same four steps in
|
||||
// web/src/App.jsx toAscii; the two must produce identical output or lookups miss.
|
||||
func ToAscii(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
@@ -44,7 +37,7 @@ func ToAscii(s string) string {
|
||||
b.Grow(len(decomposed))
|
||||
for _, r := range decomposed {
|
||||
if r >= 0x0300 && r <= 0x036F {
|
||||
continue // combining mark, in the range Rust drops
|
||||
continue // combining mark inside the dropped range
|
||||
}
|
||||
switch r {
|
||||
case 'đ', 'Đ':
|
||||
@@ -73,23 +66,22 @@ type ParsedRow struct {
|
||||
|
||||
// SkipReason says why a row was skipped, or SkipNone when it passed.
|
||||
//
|
||||
// The distinction is load-bearing for the printed counters, and the two
|
||||
// non-blank reasons are counted as source rows while BlankRow is not — but note
|
||||
// that split lives in the CALLER, not here. parser/src/main.rs has two call
|
||||
// sites with opposite outcomes for BlankRow: at :135-137 it returns before the
|
||||
// counter at :140, while at :151 it matches Err(BlankRow) => {} and falls
|
||||
// through to transform and insert. The build loop must reproduce both.
|
||||
// The distinction is load-bearing for the printed counters: the two non-blank
|
||||
// reasons count as source rows while SkipBlankRow does not. That split lives in
|
||||
// the CALLER, not here — the build loop drops blank rows before the source-row
|
||||
// counter and, at the later switch, lets SkipBlankRow fall through to transform
|
||||
// and insert. Both call sites have to stay as they are.
|
||||
type SkipReason int
|
||||
|
||||
const (
|
||||
SkipNone SkipReason = iota
|
||||
// SkipBlankRow: row is fully blank (2017-old2 only, checked before the
|
||||
// source-row counter).
|
||||
// SkipBlankRow: row is fully blank, and strip_blank_rows is on. Checked
|
||||
// before the source-row counter.
|
||||
SkipBlankRow
|
||||
// SkipEmptyField: so_bao_danh or ho_ten empty/missing.
|
||||
SkipEmptyField
|
||||
// SkipNonNumericSbd: so_bao_danh contains non-digit characters
|
||||
// (2017-old / 2017-old2 guard).
|
||||
// SkipNonNumericSbd: so_bao_danh contains non-digit characters, and
|
||||
// require_numeric_sbd is on.
|
||||
SkipNonNumericSbd
|
||||
)
|
||||
|
||||
@@ -109,10 +101,10 @@ func (s SkipReason) String() string {
|
||||
|
||||
// ValidateRow checks a row against the dataset's validation rules.
|
||||
//
|
||||
// Signature mirrors transform.rs:101-107, taking stripBlankRows and allBlank
|
||||
// explicitly; a shorter signature could not express both blank-row paths.
|
||||
// stripBlankRows and allBlank are passed explicitly because a shorter signature
|
||||
// could not express both blank-row paths.
|
||||
func ValidateRow(hoTen, soBaoDanh string, cfg *config.ValidationCfg, stripBlankRows, allBlank bool) SkipReason {
|
||||
// 2017-old2: skip fully blank rows BEFORE counting source rows.
|
||||
// Skip fully blank rows BEFORE counting source rows.
|
||||
if stripBlankRows && allBlank {
|
||||
return SkipBlankRow
|
||||
}
|
||||
@@ -128,11 +120,11 @@ func ValidateRow(hoTen, soBaoDanh string, cfg *config.ValidationCfg, stripBlankR
|
||||
return SkipNone
|
||||
}
|
||||
|
||||
// allASCIIDigits mirrors Rust's chars().all(|c| c.is_ascii_digit()).
|
||||
// allASCIIDigits reports whether every byte of s is an ASCII digit.
|
||||
//
|
||||
// Deliberately not strconv.Atoi: Atoi accepts a leading sign, so "+123" would
|
||||
// pass a check Rust rejects. Empty input returns true, matching Rust's all() on
|
||||
// an empty iterator — the empty case is caught earlier by RequireNonemptySbd.
|
||||
// pass a check that must reject it. Empty input returns true; the empty case is
|
||||
// caught earlier by RequireNonemptySbd.
|
||||
func allASCIIDigits(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] < '0' || s[i] > '9' {
|
||||
@@ -146,7 +138,7 @@ func allASCIIDigits(s string) bool {
|
||||
//
|
||||
// Every one of the 16 patterns runs against every dataset; a subject absent from
|
||||
// a given exam year never matches and stays NULL. Matching is unanchored
|
||||
// first-match, like Rust's Regex::captures.
|
||||
// first-match.
|
||||
func ParseScores(diemThi string) map[string]float64 {
|
||||
out := make(map[string]float64)
|
||||
if diemThi == "" {
|
||||
@@ -161,8 +153,8 @@ func ParseScores(diemThi string) map[string]float64 {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Unreachable given the pattern shape, but kept for parity with
|
||||
// transform.rs:136 (is_finite).
|
||||
// Unreachable given the pattern shape; kept so a widened pattern can
|
||||
// never write Inf or NaN into a score column.
|
||||
if math.IsInf(v, 0) || math.IsNaN(v) {
|
||||
continue
|
||||
}
|
||||
@@ -172,21 +164,20 @@ func ParseScores(diemThi string) map[string]float64 {
|
||||
}
|
||||
|
||||
// ErrNoColumns is returned when the fixed-column path is used on a config that
|
||||
// has no columns: mapping. Rust panics here via .expect() (transform.rs:162);
|
||||
// returning an error is the Go-idiomatic equivalent and is unreachable in
|
||||
// practice, since only the non-2016 path calls this.
|
||||
// has no columns: mapping. Unreachable in practice, since only the non-2016 path
|
||||
// calls TransformRow.
|
||||
var ErrNoColumns = errors.New("transform: config has no columns mapping")
|
||||
|
||||
// TransformRow extracts one row into a ParsedRow using fixed column indices,
|
||||
// the 2017-family path. 2016 uses runtime format detection instead.
|
||||
// the 2017 path. 2016 uses runtime format detection instead.
|
||||
func TransformRow(raw []reader.Cell, cfg *config.DatasetConfig) (*ParsedRow, error) {
|
||||
cols := cfg.Columns
|
||||
if cols == nil {
|
||||
return nil, ErrNoColumns
|
||||
}
|
||||
|
||||
// Trimmed accessor, mirroring the closure at transform.rs:163-167. Out-of-range
|
||||
// indices yield "" rather than an error, matching unwrap_or_default().
|
||||
// Trimmed accessor. Out-of-range indices yield "" rather than an error, so a
|
||||
// short row produces empty fields instead of failing.
|
||||
get := func(idx int) string {
|
||||
if idx < 0 || idx >= len(raw) {
|
||||
return ""
|
||||
@@ -198,9 +189,9 @@ func TransformRow(raw []reader.Cell, cfg *config.DatasetConfig) (*ParsedRow, err
|
||||
ngaySinh := get(cols.NgaySinh)
|
||||
soBaoDanh := get(cols.SoBaoDanh)
|
||||
|
||||
// diem_thi is read WITHOUT trimming (transform.rs:172-175), unlike the three
|
||||
// fields above. Harmless because the score patterns are unanchored, but it is
|
||||
// the shipped behaviour — do not "tidy" it.
|
||||
// diem_thi is read WITHOUT trimming, unlike the three fields above. Harmless
|
||||
// because the score patterns are unanchored, but it is the shipped behaviour
|
||||
// — do not "tidy" it.
|
||||
diemThi := ""
|
||||
if cols.DiemThi >= 0 && cols.DiemThi < len(raw) {
|
||||
diemThi = raw[cols.DiemThi].Str
|
||||
|
||||
Reference in new issue
Block a user