mirror of
https://github.com/tiennm99/thptqg.git
synced 2026-10-15 22:15:13 +00:00
refactor: one directory per pipeline stage, and an assembler to drive them
The repository now reads as the pipeline it is: crawler fetches, parser converts, assembler verifies and publishes, with data/ and web/ as the stores they hand work through. go-parser is renamed parser now that there is no other. The assembler replaces build-db.js and assemble-site.js. It compiles the parser, builds and verifies each database, compresses it, runs the Vite build and assembles _site — one command, and the only place that knows the order. It also closes a real hole: nothing previously asserted that a database reached the site. An empty staging directory assembled happily, so every page rendered, every query 404d and CI stayed green. The row-count and size guards could not catch that, since they only run when a database was built at all. Removing Node from the root forced the dataset list out of web/src/datasets.js, which the assembler cannot import. datasets.json is now the registry both sides read — JSON because Go and the browser both parse it without a dependency — while presentation stays in the web app, keyed by id and cross-checked against the registry so a half-added dataset fails instead of half-working. Guards verified by making each one fail: a missing database, and an expected row count one higher than the truth.
This commit is contained in:
tiennm99 committed
1 parent
b04f9844f9
commit
c359a0b444
61 files changed
+1341
-480
No files matched your search
@@ -0,0 +1,65 @@
|
||||
package transform_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
_ "github.com/tiennm99/thptqg/parser/internal/sqlitedb"
|
||||
"github.com/tiennm99/thptqg/parser/internal/transform"
|
||||
)
|
||||
|
||||
// TestToAsciiAgainstRustOutput cross-checks ToAscii against Rust on real data.
|
||||
//
|
||||
// A Rust-built database is its own oracle: every row carries ho_ten alongside
|
||||
// the ho_ten_ascii that Rust derived from it, so the whole table is a
|
||||
// name -> expected-slug corpus far broader than the 20 hand-picked unit cases.
|
||||
//
|
||||
// Point it at a Rust-built database:
|
||||
//
|
||||
// GO_PARSER_RUST_DB=/tmp/rust-2016.db go test ./internal/transform/
|
||||
//
|
||||
// Skips when unset, so the default suite stays hermetic.
|
||||
func TestToAsciiAgainstRustOutput(t *testing.T) {
|
||||
path := os.Getenv("GO_PARSER_RUST_DB")
|
||||
if path == "" {
|
||||
t.Skip("GO_PARSER_RUST_DB not set; skipping cross-check against Rust output")
|
||||
}
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Skipf("GO_PARSER_RUST_DB=%s not readable: %v", path, err)
|
||||
}
|
||||
|
||||
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
|
||||
if err != nil {
|
||||
t.Fatalf("open %s: %v", path, err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
rows, err := db.Query("SELECT ho_ten, ho_ten_ascii FROM student")
|
||||
if err != nil {
|
||||
t.Fatalf("query: %v", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var checked, bad int
|
||||
for rows.Next() {
|
||||
var name, rustAscii string
|
||||
if err := rows.Scan(&name, &rustAscii); err != nil {
|
||||
t.Fatalf("scan: %v", err)
|
||||
}
|
||||
checked++
|
||||
if got := transform.ToAscii(name); got != rustAscii {
|
||||
bad++
|
||||
if bad <= 5 {
|
||||
t.Errorf("ToAscii(%q)\n rust = %q\n go = %q", name, rustAscii, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
t.Fatalf("iterate: %v", err)
|
||||
}
|
||||
if checked == 0 {
|
||||
t.Fatal("database contained no rows")
|
||||
}
|
||||
t.Logf("compared %d real names, %d mismatches", checked, bad)
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
// Package transform performs row transformation: ASCII normalisation, score
|
||||
// regex parsing, and validation — a port of parser/src/transform.rs.
|
||||
//
|
||||
// ToAscii replicates build-lib.js toAscii exactly:
|
||||
//
|
||||
// str.normalize("NFD").replace(/[̀-ͯ]/g,"").replace(/đ/gi,"d").toLowerCase()
|
||||
package transform
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
|
||||
"github.com/tiennm99/thptqg/parser/internal/config"
|
||||
"github.com/tiennm99/thptqg/parser/internal/reader"
|
||||
"github.com/tiennm99/thptqg/parser/internal/schema"
|
||||
)
|
||||
|
||||
// ToAscii normalises a Vietnamese name to an ASCII slug.
|
||||
//
|
||||
// 1. NFD decompose (splits base + combining diacritics)
|
||||
// 2. Drop combining marks in U+0300..U+036F
|
||||
// 3. Replace đ/Đ with d (NFD does not decompose them)
|
||||
// 4. Lowercase
|
||||
//
|
||||
// Step 2 filters a LITERAL CODEPOINT RANGE, not a Unicode category. The inline
|
||||
// comment at transform.rs:53 says "Unicode category M", but the code at :56 is
|
||||
// the specification and it checks '\u{0300}'..='\u{036f}'. unicode.Is(unicode.Mn, r)
|
||||
// is strictly broader and would strip marks Rust keeps, silently changing
|
||||
// ho_ten_ascii — the column the site's accent-insensitive search runs on.
|
||||
//
|
||||
// Step order matters and mirrors transform.rs:54-63: the đ/Đ replacement happens
|
||||
// before lowercasing.
|
||||
func ToAscii(s string) string {
|
||||
if s == "" {
|
||||
return ""
|
||||
}
|
||||
decomposed := norm.NFD.String(s)
|
||||
|
||||
var b strings.Builder
|
||||
b.Grow(len(decomposed))
|
||||
for _, r := range decomposed {
|
||||
if r >= 0x0300 && r <= 0x036F {
|
||||
continue // combining mark, in the range Rust drops
|
||||
}
|
||||
switch r {
|
||||
case 'đ', 'Đ':
|
||||
b.WriteByte('d')
|
||||
default:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return strings.ToLower(b.String())
|
||||
}
|
||||
|
||||
// ParsedRow is one row ready for insertion.
|
||||
type ParsedRow struct {
|
||||
SoBaoDanh string
|
||||
HoTen string
|
||||
HoTenAscii string
|
||||
NgaySinh *string
|
||||
// TenCumThi is 2016 only: examination cluster name (TEN_CUMTHI column).
|
||||
TenCumThi *string
|
||||
// GioiTinh is 2016 only: gender, normalised to "Nam"/"Nữ" or nil.
|
||||
GioiTinh *string
|
||||
// Scores maps subject field -> value. Absent subjects are simply missing and
|
||||
// bind NULL.
|
||||
Scores map[string]float64
|
||||
}
|
||||
|
||||
// SkipReason says why a row was skipped, or SkipNone when it passed.
|
||||
//
|
||||
// The distinction is load-bearing for the printed counters, and the two
|
||||
// non-blank reasons are counted as source rows while BlankRow is not — but note
|
||||
// that split lives in the CALLER, not here. parser/src/main.rs has two call
|
||||
// sites with opposite outcomes for BlankRow: at :135-137 it returns before the
|
||||
// counter at :140, while at :151 it matches Err(BlankRow) => {} and falls
|
||||
// through to transform and insert. The build loop must reproduce both.
|
||||
type SkipReason int
|
||||
|
||||
const (
|
||||
SkipNone SkipReason = iota
|
||||
// SkipBlankRow: row is fully blank (2017-old2 only, checked before the
|
||||
// source-row counter).
|
||||
SkipBlankRow
|
||||
// SkipEmptyField: so_bao_danh or ho_ten empty/missing.
|
||||
SkipEmptyField
|
||||
// SkipNonNumericSbd: so_bao_danh contains non-digit characters
|
||||
// (2017-old / 2017-old2 guard).
|
||||
SkipNonNumericSbd
|
||||
)
|
||||
|
||||
func (s SkipReason) String() string {
|
||||
switch s {
|
||||
case SkipNone:
|
||||
return "none"
|
||||
case SkipBlankRow:
|
||||
return "blank_row"
|
||||
case SkipEmptyField:
|
||||
return "empty_field"
|
||||
case SkipNonNumericSbd:
|
||||
return "non_numeric_sbd"
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
// ValidateRow checks a row against the dataset's validation rules.
|
||||
//
|
||||
// Signature mirrors transform.rs:101-107, taking stripBlankRows and allBlank
|
||||
// explicitly; a shorter signature could not express both blank-row paths.
|
||||
func ValidateRow(hoTen, soBaoDanh string, cfg *config.ValidationCfg, stripBlankRows, allBlank bool) SkipReason {
|
||||
// 2017-old2: skip fully blank rows BEFORE counting source rows.
|
||||
if stripBlankRows && allBlank {
|
||||
return SkipBlankRow
|
||||
}
|
||||
if cfg.RequireNonemptySbd && soBaoDanh == "" {
|
||||
return SkipEmptyField
|
||||
}
|
||||
if cfg.RequireNonemptyName && hoTen == "" {
|
||||
return SkipEmptyField
|
||||
}
|
||||
if cfg.RequireNumericSbd && !allASCIIDigits(soBaoDanh) {
|
||||
return SkipNonNumericSbd
|
||||
}
|
||||
return SkipNone
|
||||
}
|
||||
|
||||
// allASCIIDigits mirrors Rust's chars().all(|c| c.is_ascii_digit()).
|
||||
//
|
||||
// Deliberately not strconv.Atoi: Atoi accepts a leading sign, so "+123" would
|
||||
// pass a check Rust rejects. Empty input returns true, matching Rust's all() on
|
||||
// an empty iterator — the empty case is caught earlier by RequireNonemptySbd.
|
||||
func allASCIIDigits(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] < '0' || s[i] > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ParseScores extracts subject scores from a DIEM_THI cell.
|
||||
//
|
||||
// Every one of the 16 patterns runs against every dataset; a subject absent from
|
||||
// a given exam year never matches and stays NULL. Matching is unanchored
|
||||
// first-match, like Rust's Regex::captures.
|
||||
func ParseScores(diemThi string) map[string]float64 {
|
||||
out := make(map[string]float64)
|
||||
if diemThi == "" {
|
||||
return out
|
||||
}
|
||||
for field, re := range schema.ScorePatterns {
|
||||
m := re.FindStringSubmatch(diemThi)
|
||||
if m == nil {
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseFloat(m[1], 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// Unreachable given the pattern shape, but kept for parity with
|
||||
// transform.rs:136 (is_finite).
|
||||
if math.IsInf(v, 0) || math.IsNaN(v) {
|
||||
continue
|
||||
}
|
||||
out[field] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ErrNoColumns is returned when the fixed-column path is used on a config that
|
||||
// has no columns: mapping. Rust panics here via .expect() (transform.rs:162);
|
||||
// returning an error is the Go-idiomatic equivalent and is unreachable in
|
||||
// practice, since only the non-2016 path calls this.
|
||||
var ErrNoColumns = errors.New("transform: config has no columns mapping")
|
||||
|
||||
// TransformRow extracts one row into a ParsedRow using fixed column indices,
|
||||
// the 2017-family path. 2016 uses runtime format detection instead.
|
||||
func TransformRow(raw []reader.Cell, cfg *config.DatasetConfig) (*ParsedRow, error) {
|
||||
cols := cfg.Columns
|
||||
if cols == nil {
|
||||
return nil, ErrNoColumns
|
||||
}
|
||||
|
||||
// Trimmed accessor, mirroring the closure at transform.rs:163-167. Out-of-range
|
||||
// indices yield "" rather than an error, matching unwrap_or_default().
|
||||
get := func(idx int) string {
|
||||
if idx < 0 || idx >= len(raw) {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(raw[idx].Str)
|
||||
}
|
||||
|
||||
hoTen := get(cols.HoTen)
|
||||
ngaySinh := get(cols.NgaySinh)
|
||||
soBaoDanh := get(cols.SoBaoDanh)
|
||||
|
||||
// diem_thi is read WITHOUT trimming (transform.rs:172-175), unlike the three
|
||||
// fields above. Harmless because the score patterns are unanchored, but it is
|
||||
// the shipped behaviour — do not "tidy" it.
|
||||
diemThi := ""
|
||||
if cols.DiemThi >= 0 && cols.DiemThi < len(raw) {
|
||||
diemThi = raw[cols.DiemThi].Str
|
||||
}
|
||||
|
||||
var ngaySinhOpt *string
|
||||
if ngaySinh != "" {
|
||||
ngaySinhOpt = &ngaySinh
|
||||
}
|
||||
|
||||
return &ParsedRow{
|
||||
SoBaoDanh: soBaoDanh,
|
||||
HoTen: hoTen,
|
||||
HoTenAscii: ToAscii(hoTen),
|
||||
NgaySinh: ngaySinhOpt,
|
||||
TenCumThi: nil,
|
||||
GioiTinh: nil,
|
||||
Scores: ParseScores(diemThi),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,283 @@
|
||||
package transform
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/tiennm99/thptqg/parser/internal/config"
|
||||
"github.com/tiennm99/thptqg/parser/internal/reader"
|
||||
)
|
||||
|
||||
// Ports every test in parser/src/transform.rs's test module (:201-409) — 29 in
|
||||
// total, not the 20 in the :213-315 range, which covers only ToAscii. The nine
|
||||
// outside that range are the ParseScores and ValidateRow cases, i.e. exactly the
|
||||
// behaviours this package's traps concern.
|
||||
|
||||
// --- ToAscii: the 20 cases at transform.rs:213-315 ---
|
||||
|
||||
func TestToAscii(t *testing.T) {
|
||||
cases := []struct{ name, in, want string }{
|
||||
{"plain_latin", "Nguyen Van A", "nguyen van a"},
|
||||
{"nguyen_thi_hoa", "Nguyễn Thị Hoa", "nguyen thi hoa"},
|
||||
{"tran_van_duc", "Trần Văn Đức", "tran van duc"},
|
||||
{"le_thi_my_duyen", "Lê Thị Mỹ Duyên", "le thi my duyen"},
|
||||
{"pham_thi_lan", "Phạm Thị Lan", "pham thi lan"},
|
||||
{"bui_thi_thu", "Bùi Thị Thu", "bui thi thu"},
|
||||
{"hoang_van_truong", "Hoàng Văn Trường", "hoang van truong"},
|
||||
{"do_thi_ngan", "Đỗ Thị Ngân", "do thi ngan"},
|
||||
{"nguyen_van_khanh", "Nguyễn Văn Khánh", "nguyen van khanh"},
|
||||
{"trinh_thi_bich_ngoc", "Trịnh Thị Bích Ngọc", "trinh thi bich ngoc"},
|
||||
{"vu_thi_dieu", "Vũ Thị Diệu", "vu thi dieu"},
|
||||
{"nguyen_thi_tuong_vi", "Nguyễn Thị Tường Vi", "nguyen thi tuong vi"},
|
||||
{"lowercase_d_stroke", "đặng thị hằng", "dang thi hang"},
|
||||
{"uppercase_d_stroke", "ĐẶNG THỊ HẰNG", "dang thi hang"},
|
||||
{"mixed_case", "NGUYỄN VĂN AN", "nguyen van an"},
|
||||
{"tran_thi_kim_anh", "Trần Thị Kim Anh", "tran thi kim anh"},
|
||||
{"nguyen_thi_phuong_thao", "Nguyễn Thị Phương Thảo", "nguyen thi phuong thao"},
|
||||
{"le_van_long", "Lê Văn Long", "le van long"},
|
||||
{"vo_thi_xuan_mai", "Võ Thị Xuân Mai", "vo thi xuan mai"},
|
||||
{"empty_string", "", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
if got := ToAscii(c.in); got != c.want {
|
||||
t.Errorf("ToAscii(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestToAsciiUsesLiteralRangeNotUnicodeMn guards the highest-value trap in this
|
||||
// package. transform.rs:56 filters the literal range U+0300..U+036F; the inline
|
||||
// comment at :53 calls it "Unicode category M", but the code is the spec.
|
||||
// unicode.Mn is strictly broader, so using it would strip marks Rust keeps.
|
||||
// U+0654 (ARABIC HAMZA ABOVE) is in Mn but outside the range: Rust keeps it.
|
||||
func TestToAsciiUsesLiteralRangeNotUnicodeMn(t *testing.T) {
|
||||
const in = "aٔb"
|
||||
if got := ToAscii(in); got != in {
|
||||
t.Errorf("ToAscii(%q) = %q — a combining mark outside U+0300..U+036F must survive; "+
|
||||
"stripping it means unicode.Mn was used instead of the literal range", in, got)
|
||||
}
|
||||
// And a mark inside the range must be stripped.
|
||||
if got := ToAscii("áb"); got != "ab" {
|
||||
t.Errorf("ToAscii(\"a\\u0301b\") = %q, want \"ab\"", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestToAsciiDStrokeIndependentOfNFD proves the đ/Đ replacement is a separate
|
||||
// step: NFD does not decompose them, so relying on the mark filter alone loses
|
||||
// the letter entirely.
|
||||
func TestToAsciiDStrokeIndependentOfNFD(t *testing.T) {
|
||||
for _, c := range []struct{ in, want string }{
|
||||
{"đ", "d"}, {"Đ", "d"}, {"đĐ", "dd"},
|
||||
} {
|
||||
if got := ToAscii(c.in); got != c.want {
|
||||
t.Errorf("ToAscii(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- ParseScores: transform.rs:323, :332, :342 ---
|
||||
|
||||
func TestParseScoresSingle(t *testing.T) {
|
||||
s := ParseScores("Toán: 8.5")
|
||||
if v, ok := s["toan"]; !ok || v != 8.5 {
|
||||
t.Errorf("toan = %v (present=%v), want 8.5", v, ok)
|
||||
}
|
||||
if _, ok := s["ngu_van"]; ok {
|
||||
t.Error("ngu_van should be absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseScoresMultiple(t *testing.T) {
|
||||
s := ParseScores("Toán: 7.25 Ngữ văn: 6.0 Vật lí: 9")
|
||||
for field, want := range map[string]float64{"toan": 7.25, "ngu_van": 6.0, "vat_ly": 9.0} {
|
||||
if v, ok := s[field]; !ok || v != want {
|
||||
t.Errorf("%s = %v (present=%v), want %v", field, v, ok, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseScoresEmptyCell(t *testing.T) {
|
||||
if s := ParseScores(""); len(s) != 0 {
|
||||
t.Errorf("ParseScores(\"\") = %v, want empty", s)
|
||||
}
|
||||
}
|
||||
|
||||
// TestParseScoresRealCellShape uses the wide space runs seen in the corpus.
|
||||
func TestParseScoresRealCellShape(t *testing.T) {
|
||||
const cell = "Toán: 4.60 Ngữ văn: 5.50 Lịch sử: 4.50 "
|
||||
s := ParseScores(cell)
|
||||
if len(s) != 3 {
|
||||
t.Fatalf("matched %d subjects, want 3: %v", len(s), s)
|
||||
}
|
||||
if s["toan"] != 4.60 || s["ngu_van"] != 5.50 || s["lich_su"] != 4.50 {
|
||||
t.Errorf("got %v", s)
|
||||
}
|
||||
}
|
||||
|
||||
// --- ValidateRow: transform.rs:359, :365, :374, :383, :393, :400 ---
|
||||
|
||||
func defaultValidation() *config.ValidationCfg {
|
||||
return &config.ValidationCfg{
|
||||
RequireNumericSbd: false,
|
||||
RequireNonemptyName: true,
|
||||
RequireNonemptySbd: true,
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateOK(t *testing.T) {
|
||||
if r := ValidateRow("Nguyen Van A", "12345678", defaultValidation(), false, false); r != SkipNone {
|
||||
t.Errorf("got %v, want SkipNone", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateEmptySbd(t *testing.T) {
|
||||
if r := ValidateRow("Nguyen Van A", "", defaultValidation(), false, false); r != SkipEmptyField {
|
||||
t.Errorf("got %v, want SkipEmptyField", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateEmptyName(t *testing.T) {
|
||||
if r := ValidateRow("", "12345678", defaultValidation(), false, false); r != SkipEmptyField {
|
||||
t.Errorf("got %v, want SkipEmptyField", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNonNumericSbdRejected(t *testing.T) {
|
||||
v := defaultValidation()
|
||||
v.RequireNumericSbd = true
|
||||
if r := ValidateRow("Nguyen Van A", "12AB5678", v, false, false); r != SkipNonNumericSbd {
|
||||
t.Errorf("got %v, want SkipNonNumericSbd", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateNumericSbdAccepted(t *testing.T) {
|
||||
v := defaultValidation()
|
||||
v.RequireNumericSbd = true
|
||||
if r := ValidateRow("Nguyen Van A", "12345678", v, false, false); r != SkipNone {
|
||||
t.Errorf("got %v, want SkipNone", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBlankRowSkipped(t *testing.T) {
|
||||
if r := ValidateRow("", "", defaultValidation(), true, true); r != SkipBlankRow {
|
||||
t.Errorf("got %v, want SkipBlankRow", r)
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateNumericSbdIsDigitScanNotAtoi: Rust uses chars().all(is_ascii_digit),
|
||||
// which strconv.Atoi does not reproduce — Atoi accepts a leading sign, and would
|
||||
// wrongly admit "+123".
|
||||
func TestValidateNumericSbdIsDigitScanNotAtoi(t *testing.T) {
|
||||
v := defaultValidation()
|
||||
v.RequireNumericSbd = true
|
||||
for _, sbd := range []string{"+123", "-123", "12 3", "1.0", "ABC123", "１２３"} {
|
||||
if r := ValidateRow("Nguyen Van A", sbd, v, false, false); r != SkipNonNumericSbd {
|
||||
t.Errorf("ValidateRow(sbd=%q) = %v, want SkipNonNumericSbd", sbd, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateBlankRowOnlyWhenStripEnabled: with strip_blank_rows false, an
|
||||
// all-blank row falls through to the empty-field checks instead (transform.rs:109).
|
||||
func TestValidateBlankRowOnlyWhenStripEnabled(t *testing.T) {
|
||||
if r := ValidateRow("", "", defaultValidation(), false, true); r != SkipEmptyField {
|
||||
t.Errorf("got %v, want SkipEmptyField when strip_blank_rows is off", r)
|
||||
}
|
||||
}
|
||||
|
||||
// --- TransformRow ---
|
||||
|
||||
func fixedColumnCfg() *config.DatasetConfig {
|
||||
return &config.DatasetConfig{
|
||||
Columns: &config.ColumnMap{HoTen: 0, NgaySinh: 1, SoBaoDanh: 2, DiemThi: 3},
|
||||
Validation: *defaultValidation(),
|
||||
}
|
||||
}
|
||||
|
||||
func cells(vals ...string) []reader.Cell {
|
||||
out := make([]reader.Cell, len(vals))
|
||||
for i, v := range vals {
|
||||
out[i] = reader.Cell{Str: v, IsEmpty: v == ""}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestTransformRow(t *testing.T) {
|
||||
row := cells("Nguyễn Văn Đức", "04/04/1999", "51002167", "Toán: 8.5 Ngữ văn: 7")
|
||||
got, err := TransformRow(row, fixedColumnCfg())
|
||||
if err != nil {
|
||||
t.Fatalf("TransformRow: %v", err)
|
||||
}
|
||||
if got.HoTen != "Nguyễn Văn Đức" || got.HoTenAscii != "nguyen van duc" {
|
||||
t.Errorf("ho_ten=%q ascii=%q", got.HoTen, got.HoTenAscii)
|
||||
}
|
||||
if got.SoBaoDanh != "51002167" {
|
||||
t.Errorf("so_bao_danh = %q", got.SoBaoDanh)
|
||||
}
|
||||
if got.NgaySinh == nil || *got.NgaySinh != "04/04/1999" {
|
||||
t.Errorf("ngay_sinh = %v", got.NgaySinh)
|
||||
}
|
||||
// 2016-only columns are never populated on the fixed-column path.
|
||||
if got.TenCumThi != nil || got.GioiTinh != nil {
|
||||
t.Error("ten_cum_thi and gioi_tinh must stay nil on the 2017 path")
|
||||
}
|
||||
if got.Scores["toan"] != 8.5 || got.Scores["ngu_van"] != 7 {
|
||||
t.Errorf("scores = %v", got.Scores)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTransformRowEmptyNgaySinhBecomesNil ports transform.rs:179-183.
|
||||
func TestTransformRowEmptyNgaySinhBecomesNil(t *testing.T) {
|
||||
got, err := TransformRow(cells("A", "", "1", ""), fixedColumnCfg())
|
||||
if err != nil {
|
||||
t.Fatalf("TransformRow: %v", err)
|
||||
}
|
||||
if got.NgaySinh != nil {
|
||||
t.Errorf("empty ngay_sinh should be nil, got %q", *got.NgaySinh)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTransformRowShortRowYieldsEmptyFields ports the unwrap_or_default()
|
||||
// behaviour at transform.rs:163-176: a row shorter than the configured indices
|
||||
// yields empty strings rather than an error.
|
||||
func TestTransformRowShortRowYieldsEmptyFields(t *testing.T) {
|
||||
got, err := TransformRow(cells("OnlyName"), fixedColumnCfg())
|
||||
if err != nil {
|
||||
t.Fatalf("TransformRow: %v", err)
|
||||
}
|
||||
if got.HoTen != "OnlyName" || got.SoBaoDanh != "" || got.NgaySinh != nil {
|
||||
t.Errorf("got ho_ten=%q sbd=%q ngay_sinh=%v", got.HoTen, got.SoBaoDanh, got.NgaySinh)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTransformRowDiemThiIsNotTrimmed pins an asymmetry that is easy to
|
||||
// "tidy away": ho_ten, ngay_sinh and so_bao_danh are trimmed through the closure
|
||||
// at transform.rs:164-168, but diem_thi is read raw at :172-175.
|
||||
func TestTransformRowDiemThiIsNotTrimmed(t *testing.T) {
|
||||
row := cells(" A ", " 01/01/2000 ", " 123 ", " Toán: 5 ")
|
||||
got, err := TransformRow(row, fixedColumnCfg())
|
||||
if err != nil {
|
||||
t.Fatalf("TransformRow: %v", err)
|
||||
}
|
||||
if got.HoTen != "A" || got.SoBaoDanh != "123" {
|
||||
t.Errorf("trimmed fields wrong: ho_ten=%q sbd=%q", got.HoTen, got.SoBaoDanh)
|
||||
}
|
||||
if got.NgaySinh == nil || *got.NgaySinh != "01/01/2000" {
|
||||
t.Errorf("ngay_sinh = %v, want trimmed", got.NgaySinh)
|
||||
}
|
||||
// Untrimmed diem_thi still parses — the regexes are unanchored.
|
||||
if got.Scores["toan"] != 5 {
|
||||
t.Errorf("scores = %v", got.Scores)
|
||||
}
|
||||
}
|
||||
|
||||
// TestTransformRowRequiresColumns: the fixed-column path is only reachable when
|
||||
// the config has a columns: mapping. Rust panics via .expect() (transform.rs:162);
|
||||
// Go returns an error instead.
|
||||
func TestTransformRowRequiresColumns(t *testing.T) {
|
||||
cfg := &config.DatasetConfig{Validation: *defaultValidation()}
|
||||
if _, err := TransformRow(cells("A", "B", "C", "D"), cfg); err == nil {
|
||||
t.Fatal("TransformRow without a columns: mapping must return an error")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user