diff --git a/.github/workflows/test-and-coverage.yml b/.github/workflows/test-and-coverage.yml index f36a45c..7ab2380 100644 --- a/.github/workflows/test-and-coverage.yml +++ b/.github/workflows/test-and-coverage.yml @@ -11,6 +11,7 @@ jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: lua: [lua=5.1, lua=5.2, lua=5.3, lua=5.4, luajit=@v2.0, luajit=@v2.1] steps: @@ -19,6 +20,13 @@ jobs: - name: Install libreadline run: sudo apt-get install -y libreadline-dev + - name: Cache Restore + uses: actions/cache/restore@v5 + with: + path: | + lua_install + key: ${{ matrix.lua }}-lua_install + - name: Install Lua (${{ matrix.lua }}) run: | pip install git+https://github.com/luarocks/hererocks @@ -27,14 +35,20 @@ jobs: - name: Install depedencies run: | - luarocks install busted + luarocks install tested luarocks install lua-cjson - luarocks install luacov luarocks install luacov-coveralls - name: Run unit tests with coverage - run: busted --verbose --coverage - + run: tested -c + + - name: Save Cache + uses: actions/cache/save@v5 + with: + path: | + lua_install + key: ${{ matrix.lua }}-lua_install + - name: Report test coverage if: success() continue-on-error: true diff --git a/tests/bad_csvs/empty_file.csv b/tests/bad_csvs/empty_file.csv new file mode 100644 index 0000000..e69de29 diff --git a/tests/bad_csvs/empty_file_newline.csv b/tests/bad_csvs/empty_file_newline.csv new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/bad_csvs/empty_file_newline.csv @@ -0,0 +1 @@ + diff --git a/tests/bad_csvs/empty_header.csv b/tests/bad_csvs/empty_header.csv new file mode 100644 index 0000000..f02adab --- /dev/null +++ b/tests/bad_csvs/empty_header.csv @@ -0,0 +1,2 @@ +a,b, +herp,derp \ No newline at end of file diff --git a/tests/bad_csvs/too_few_cols.csv b/tests/bad_csvs/too_few_cols.csv new file mode 100644 index 0000000..6d20020 --- /dev/null +++ b/tests/bad_csvs/too_few_cols.csv @@ -0,0 +1,3 @@ +a,b,c +failing,hard +man,oh,well... \ No newline at end of file diff --git a/tests/bad_csvs/too_few_cols_end.csv b/tests/bad_csvs/too_few_cols_end.csv new file mode 100644 index 0000000..2b32766 --- /dev/null +++ b/tests/bad_csvs/too_few_cols_end.csv @@ -0,0 +1,3 @@ +a,b,c +man,oh,well... +failing,hard \ No newline at end of file diff --git a/tests/bad_csvs/too_many_cols.csv b/tests/bad_csvs/too_many_cols.csv new file mode 100644 index 0000000..18f92b9 --- /dev/null +++ b/tests/bad_csvs/too_many_cols.csv @@ -0,0 +1,3 @@ +a,b,c +no,one,knows +what,am,i,doing? \ No newline at end of file diff --git a/tests/csvs/bom-os9.csv b/tests/csvs/bom-os9.csv new file mode 100644 index 0000000..3ea2148 --- /dev/null +++ b/tests/csvs/bom-os9.csv @@ -0,0 +1 @@ +a,b,c 1,2,3 4,5,ʤ \ No newline at end of file diff --git a/tests/csvs/comma_in_quotes.csv b/tests/csvs/comma_in_quotes.csv new file mode 100644 index 0000000..2aed6c1 --- /dev/null +++ b/tests/csvs/comma_in_quotes.csv @@ -0,0 +1,2 @@ +first,last,address,city,zip +John,Doe,120 any st.,"Anytown, WW",08123 \ No newline at end of file diff --git a/tests/csvs/correctness.csv b/tests/csvs/correctness.csv new file mode 100644 index 0000000..9920d35 --- /dev/null +++ b/tests/csvs/correctness.csv @@ -0,0 +1,7 @@ +Year,Make,Model,Description,Price +1997,Ford,E350,"ac, abs, moon",3000.00 +1999,Chevy,"Venture ""Extended Edition""","",4900.00 +1996,Jeep,Grand Cherokee,"MUST SELL! +air, moon roof, loaded",4799.00 +1999,Chevy,"Venture ""Extended Edition, Very Large""",,5000.00 +,,"Venture ""Extended Edition""","",4900.00 \ No newline at end of file diff --git a/tests/csvs/empty.csv b/tests/csvs/empty.csv new file mode 100644 index 0000000..7fc5cfa --- /dev/null +++ b/tests/csvs/empty.csv @@ -0,0 +1,3 @@ +a,b,c +1,"","" +2,3,4 \ No newline at end of file diff --git a/tests/csvs/empty_crlf.csv b/tests/csvs/empty_crlf.csv new file mode 100644 index 0000000..a84a780 --- /dev/null +++ b/tests/csvs/empty_crlf.csv @@ -0,0 +1,3 @@ +a,b,c +1,"","" +2,3,4 \ No newline at end of file diff --git a/tests/csvs/empty_no_newline.csv b/tests/csvs/empty_no_newline.csv new file mode 100644 index 0000000..b240a12 --- /dev/null +++ b/tests/csvs/empty_no_newline.csv @@ -0,0 +1,2 @@ +a,b,c +1,"","" \ No newline at end of file diff --git a/tests/csvs/empty_no_quotes.csv b/tests/csvs/empty_no_quotes.csv new file mode 100644 index 0000000..70c1a09 --- /dev/null +++ b/tests/csvs/empty_no_quotes.csv @@ -0,0 +1,2 @@ +a,b,c +1,, \ No newline at end of file diff --git a/tests/csvs/escaped_quotes.csv b/tests/csvs/escaped_quotes.csv new file mode 100644 index 0000000..d4a05cb --- /dev/null +++ b/tests/csvs/escaped_quotes.csv @@ -0,0 +1,3 @@ +a,b +1,"ha ""ha"" ha" +3,4 diff --git a/tests/csvs/escaped_quotes_in_header.csv b/tests/csvs/escaped_quotes_in_header.csv new file mode 100644 index 0000000..2223687 --- /dev/null +++ b/tests/csvs/escaped_quotes_in_header.csv @@ -0,0 +1,3 @@ +"li""on",tiger,"be""ar" +1,2,3 +5,6,7 \ No newline at end of file diff --git a/tests/csvs/json.csv b/tests/csvs/json.csv new file mode 100644 index 0000000..f2c2812 --- /dev/null +++ b/tests/csvs/json.csv @@ -0,0 +1,2 @@ +key,val +1,"{""type"": ""Point"", ""coordinates"": [102.0, 0.5]}" diff --git a/tests/csvs/json_no_newline.csv b/tests/csvs/json_no_newline.csv new file mode 100644 index 0000000..46ae1fd --- /dev/null +++ b/tests/csvs/json_no_newline.csv @@ -0,0 +1,2 @@ +key,val +1,"{""type"": ""Point"", ""coordinates"": [102.0, 0.5]}" \ No newline at end of file diff --git a/tests/csvs/missing_keys.csv b/tests/csvs/missing_keys.csv new file mode 100644 index 0000000..9d0480c --- /dev/null +++ b/tests/csvs/missing_keys.csv @@ -0,0 +1,4 @@ +a,b,c +1,2,3 +10,20, +100,,300 diff --git a/tests/csvs/newlines.csv b/tests/csvs/newlines.csv new file mode 100644 index 0000000..e034f33 --- /dev/null +++ b/tests/csvs/newlines.csv @@ -0,0 +1,5 @@ +a,b,c +1,2,3 +"Once upon +a time",5,6 +7,8,9 diff --git a/tests/csvs/newlines_crlf.csv b/tests/csvs/newlines_crlf.csv new file mode 100644 index 0000000..dc07eb4 --- /dev/null +++ b/tests/csvs/newlines_crlf.csv @@ -0,0 +1,5 @@ +a,b,c +1,2,3 +"Once upon +a time",5,6 +7,8,9 diff --git a/tests/csvs/os9.csv b/tests/csvs/os9.csv new file mode 100644 index 0000000..4f06168 --- /dev/null +++ b/tests/csvs/os9.csv @@ -0,0 +1 @@ +a,b,c 1,2,3 4,5,ʤ \ No newline at end of file diff --git a/tests/csvs/quotes_and_newlines.csv b/tests/csvs/quotes_and_newlines.csv new file mode 100644 index 0000000..0d911b8 --- /dev/null +++ b/tests/csvs/quotes_and_newlines.csv @@ -0,0 +1,5 @@ +a,b +1,"ha +""ha"" +ha" +3,4 diff --git a/tests/csvs/quotes_non_escaped.csv b/tests/csvs/quotes_non_escaped.csv new file mode 100644 index 0000000..c1f7447 --- /dev/null +++ b/tests/csvs/quotes_non_escaped.csv @@ -0,0 +1,2 @@ +"Country","City","AccentCity","Region" +af,dekh"iykh'ya,Dekh"iykh'ya,13 \ No newline at end of file diff --git a/tests/csvs/simple.csv b/tests/csvs/simple.csv new file mode 100644 index 0000000..bfde6bf --- /dev/null +++ b/tests/csvs/simple.csv @@ -0,0 +1,2 @@ +a,b,c +1,2,3 diff --git a/tests/csvs/simple_crlf.csv b/tests/csvs/simple_crlf.csv new file mode 100644 index 0000000..ea16e67 --- /dev/null +++ b/tests/csvs/simple_crlf.csv @@ -0,0 +1,2 @@ +a,b,c +1,2,3 diff --git a/tests/csvs/utf8.csv b/tests/csvs/utf8.csv new file mode 100644 index 0000000..4504964 --- /dev/null +++ b/tests/csvs/utf8.csv @@ -0,0 +1,3 @@ +a,b,c +1,2,3 +4,5,ʤ \ No newline at end of file diff --git a/tests/dynamic_features_test.lua b/tests/dynamic_features_test.lua new file mode 100644 index 0000000..d412ea5 --- /dev/null +++ b/tests/dynamic_features_test.lua @@ -0,0 +1,614 @@ +local ftcsv = require "ftcsv" +local tested = require("tested") + +local BOM = {["NO BOM"] = "", ["BOM"] = string.char(239, 187, 191)} +local newlines = {["LF"] = "\n", ["CRLF"] = "\r\n", ["CR"] = "\r"} +local endlines = {"NONE", "NEWLINE"} +local quotes = {["NO QUOTES"] = "", ["DOUBLE QUOTES"] = '"'} + + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle loading from string (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"a", "b", "c"} + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle renaming fields (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"d", "e", "f"} + local expected = {} + expected[1] = {} + expected[1].d = "apple" + expected[1].e = "banana" + expected[1].f = "carrot" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "f"}} + local actual, actualHeaders = ftcsv.parse(defaultString, options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle renaming fields to the same out value (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"d", "e"} + local expected = {} + expected[1] = {} + expected[1].d = "apple" + expected[1].e = "carrot" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "e"}} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle keeping only a few fields (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"a", "b"} + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, fieldsToKeep={"a", "b"}} + local actual, actualHeaders = ftcsv.parse(defaultString, options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle only keeping a few fields with a rename to an existing field (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"a", "b"} + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "carrot" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, rename={["c"] = "b"}, fieldsToKeep={"a","b"}} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle only keeping a few fields with a rename to a new field (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"a", "f"} + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].f = "carrot" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, rename={["c"] = "f"}, fieldsToKeep={"a","f"}} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should apply a function via headerFunc (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"A", "B", "C"} + local expected = {} + expected[1] = {} + expected[1].A = "apple" + expected[1].B = "banana" + expected[1].C = "carrot" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, headerFunc=string.upper} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should apply a function via headerFunc with rename and fieldsToKeep (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"A", "F"} + local expected = {} + expected[1] = {} + expected[1].A = "apple" + expected[1].F = "carrot" + + local defaultString = "%s`a`,`b`,`c`%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, rename={["c"] = "f"}, fieldsToKeep={"A","F"}, headerFunc=string.upper} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for _, endline in ipairs(endlines) do + local name = "should handle escaped doublequotes (%s + %s) EOF: %s" + tested.test(name:format(bom, newline, endline), function() + local expectedHeaders = {"a", "b", "c"} + local expected = {} + expected[1] = {} + expected[1].a = '"apple"' + expected[1].b = '"banana"' + expected[1].c = '"carrot"' + + local defaultString = '%s"a","b","c"%s"""apple""","""banana""","""carrot"""%s' + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end +end + +-- HEADERLESS TESTS START HERE + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle files without headers (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {1, 2, 3} + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + expected[2] = {} + expected[2][1] = "diamond" + expected[2][2] = "emerald" + expected[2][3] = "pearl" + + local defaultString = "%s`apple`,`banana`,`carrot`%s`diamond`,`emerald`,`pearl`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, headers=false} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle files without headers and with one row (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {1, 2, 3} + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + + local defaultString = "%s`apple`,`banana`,`carrot`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, "") + else + defaultString = defaultString:format(i, j) + end + + local options = {loadFromString=true, headers=false} + local actual, actualHeaders = ftcsv.parse(defaultString, options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle renaming fields from files without headers (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"a", "b", "c"} + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + expected[2].c = "pearl" + + local defaultString = "%s`apple`,`banana`,`carrot`%s`diamond`,`emerald`,`pearl`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, headers=false, rename={"a","b","c"}} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle renaming fields from files without headers and only keeping a few fields (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"a", "b"} + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + + local defaultString = "%s`apple`,`banana`,`carrot`%s`diamond`,`emerald`,`pearl`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, headers=false, rename={"a","b","c"}, fieldsToKeep={"a","b"}} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for quote, k in pairs(quotes) do + for _, endline in ipairs(endlines) do + local name = "should handle if the number of renames doesn't equal the number of fields (%s + %s + %s) EOF: %s" + tested.test(name:format(bom, newline, quote, endline), function() + local expectedHeaders = {"a", "b"} + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + + local defaultString = "%s`apple`,`banana`,`carrot`%s`diamond`,`emerald`,`pearl`%s" + defaultString = defaultString:gsub("`", k) + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, headers=false, rename={"a","b"}, fieldsToKeep={"a","b"}} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end + end +end + +for bom, i in pairs(BOM) do + for newline, j in pairs(newlines) do + for _, endline in ipairs(endlines) do + local name = "should handle ignoring quotes (%s + %s) EOF: %s" + tested.test(name:format(bom, newline, endline), function() + local expectedHeaders = {"a", "b", "c"} + local expected = {} + expected[1] = {} + expected[1].a = '"apple"' + expected[1].b = '"banana"' + expected[1].c = '"carrot"' + + local defaultString = '%sa,b,c%s"apple","banana","carrot"%s' + + if endline == "NONE" then + defaultString = defaultString:format(i, j, "") + else + defaultString = defaultString:format(i, j, j) + end + + local options = {loadFromString=true, ignoreQuotes=true} + local actual, actualHeaders = ftcsv.parse(defaultString, ",", options) + tested.assert({ + should="values be the same", + expected=expected, + actual=actual + }) + tested.assert({ + should="headers be the same", + expected=expectedHeaders, + actual=actualHeaders + }) + end) + end + end +end + +return tested \ No newline at end of file diff --git a/tests/error_test.lua b/tests/error_test.lua new file mode 100644 index 0000000..ce6b62b --- /dev/null +++ b/tests/error_test.lua @@ -0,0 +1,96 @@ +local ftcsv = require('ftcsv') +local tested = require("tested") + +local files = { + {"empty_file", "ftcsv: Cannot parse an empty file"}, + {"empty_file_newline", "ftcsv: Cannot parse a file which contains empty headers"}, + {"empty_header", "ftcsv: Cannot parse a file which contains empty headers"}, + {"too_few_cols", "ftcsv: too few columns in row 1"}, + {"too_few_cols_end", "ftcsv: too few columns in row 2"}, + {"too_many_cols", "ftcsv: too many columns in row 2"}, + {"dne", "ftcsv: File not found at spec/bad_csvs/dne.csv"} +} + +tested.test("csv decode error", function() + for _, value in ipairs(files) do + local filename = "spec/bad_csvs/" .. value[1] .. ".csv" + tested.assert_throws_exception({ + given=filename, + should="throw specific exception", + expected=value[2], + actual=function() ftcsv.parse(filename, ",") end + }) + end +end) + +tested.test("no headers or renaming", function() + local test = function() + local options = {loadFromString=true, headers=false, fieldsToKeep={1, 2}} + ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", ">", options) + end + tested.assert_throws_exception({ + given="no headers and no renaming takes place", + expected="ftcsv: fieldsToKeep only works with header-less files when using the 'rename' functionality", + actual=test + }) + +end) + + +tested.test("encode error out with missing field", function() + local encodeThis = { + {a = 'herp1', b = 'derp1'}, + {a = 'herp2', b = 'derp2'}, + {a = 'herp3', b = 'derp3'}, + } + + local test = function() + ftcsv.encode(encodeThis, ">", {fieldsToKeep={"c"}}) + end + + tested.assert_throws_exception({ + given="specify a field that doesn't exist during encode", + expected="ftcsv: the field 'c' doesn't exist in the inputTable", + actual=test + }) +end) + +tested.test("parseLine and loadFromString", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("a,b,c\n1,2,3", ",", {loadFromString=true}) do + parse[i] = line + end + return parse + end + + tested.assert_throws_exception({ + given="parseLine and loadFromString", + expected="ftcsv: parseLine currently doesn't support loading from string", + actual=test + }) +end) + +tested.test("missing quotes", function() + local test = function() + local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', ",", {loadFromString=true}) + end + tested.assert_throws_exception({ + given="missing quotes", + expected="ftcsv: can't find closing quote in row 1. Try running with the option ignoreQuotes=true if the source incorrectly uses quotes.", + actual=test + }) +end) + +tested.test("buffersize without parseLine", function() + local test = function() + local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', ",", {loadFromString=true, bufferSize=34}) + end + tested.assert_throws_exception({ + should="error if bufferSize is set when parsing entire files", + expected="ftcsv: bufferSize can only be specified using 'parseLine'. When using 'parse', the entire file is read into memory", + actual=test, + }) +end) + +return tested \ No newline at end of file diff --git a/tests/feature_test.lua b/tests/feature_test.lua new file mode 100644 index 0000000..c6f41ab --- /dev/null +++ b/tests/feature_test.lua @@ -0,0 +1,611 @@ +local ftcsv = require('ftcsv') +local tested = require("tested") + + +tested.test("should handle loading from string", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse("a,b,c\napple,banana,carrot", ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle crlf loading from string", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle cr loading from string", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse("a,b,c\rapple,banana,carrot", ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle quotes loading from string", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse('"a","b","c"\n"apple","banana","carrot"', ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle doublequotes loading from string", function() + local expected = {} + expected[1] = {} + expected[1].a = '"apple"' + expected[1].b = '"banana"' + expected[1].c = '"carrot"' + local actual = ftcsv.parse('"a","b","c"\n"""apple""","""banana""","""carrot"""', ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle doublequotes loading from string", function() + local expected = {} + expected[1] = {} + expected[1].a = '"apple"' + expected[1].b = 'banana' + expected[1].c = '"carrot"' + local actual = ftcsv.parse('"a","b","c"\n"""apple""","banana","""carrot"""', ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle escaped doublequotes", function() + local expected = {} + expected[1] = {} + expected[1].a = 'A"B""C' + expected[1].b = 'A""B"C' + expected[1].c = 'A"""B""C' + local actual = ftcsv.parse('a;b;c\n"A""B""""C";"A""""B""C";"A""""""B""""C"', ";", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle escaped doublequotes with delimiter in options", function() + local expected = {} + expected[1] = {} + expected[1].a = 'A"B""C' + expected[1].b = 'A""B"C' + expected[1].c = 'A"""B""C' + local actual = ftcsv.parse('a;b;c\n"A""B""""C";"A""""B""C";"A""""""B""""C"', {loadFromString=true, delimiter=";"}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle renaming a field", function() + local expected = {} + expected[1] = {} + expected[1].d = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", ",", {loadFromString=true, rename={["a"] = "d"}}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle renaming multiple fields", function() + local expected = {} + expected[1] = {} + expected[1].d = "apple" + expected[1].e = "banana" + expected[1].f = "carrot" + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "f"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should return a table with column headers", function() + local expected = { 'd', 'e', 'f' } + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "f"}} + local _, actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle renaming multiple fields to the same out value", function() + local expected = {} + expected[1] = {} + expected[1].d = "apple" + expected[1].e = "carrot" + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "e"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot", ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle renaming multiple fields to the same out value with newline at end", function() + local expected = {} + expected[1] = {} + expected[1].d = "apple" + expected[1].e = "carrot" + local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "e"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot\r\n", ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle only keeping a few fields", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + local options = {loadFromString=true, fieldsToKeep={"a","b"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot\r\n", ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle only keeping a few fields with a rename to an existing field", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "carrot" + local options = {loadFromString=true, fieldsToKeep={"a","b"}, rename={["c"] = "b"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot\r\n", ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle only keeping a few fields with a rename to a new field", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].f = "carrot" + local options = {loadFromString=true, fieldsToKeep={"a","f"}, rename={["c"] = "f"}} + local actual = ftcsv.parse("a,b,c\r\napple,banana,carrot\r\n", ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without headers", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + expected[2] = {} + expected[2][1] = "diamond" + expected[2][2] = "emerald" + expected[2][3] = "pearl" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without headers with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "" + expected[2] = {} + expected[2][1] = "diamond" + expected[2][2] = "emerald" + expected[2][3] = "pearl" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>\ndiamond>emerald>pearl", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines)", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>carrot", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines) with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files with quotes and without (headers and newlines)", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse('"apple">"banana">"carrot"', ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files with quotes and without (headers and newlines) with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse('"apple">"banana">', ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files with quotes and without (headers and newlines)", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + expected[2] = {} + expected[2][1] = "diamond" + expected[2][2] = "emerald" + expected[2][3] = "pearl" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse('"apple">"banana">"carrot"\n"diamond">"emerald">"pearl"', ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files with quotes and without (headers and newlines) with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "" + expected[2] = {} + expected[2][1] = "diamond" + expected[2][2] = "emerald" + expected[2][3] = "pearl" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse('"apple">"banana">\n"diamond">"emerald">"pearl"', ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines) w/newline at end", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>carrot\n", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines) w/newline at end with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>\n", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines) w/crlf", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>carrot\r\n", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines) w/crlf with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>\r\n", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines) w/cr", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "carrot" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>carrot\r", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle files without (headers and newlines) w/cr with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1][1] = "apple" + expected[1][2] = "banana" + expected[1][3] = "" + local options = {loadFromString=true, headers=false} + local actual = ftcsv.parse("apple>banana>\r", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + + +tested.test("should handle only renaming fields from files without headers", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + expected[2].c = "pearl" + local options = {loadFromString=true, headers=false, rename={"a","b","c"}} + local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle only renaming fields from files without headers with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[1].c = "" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + expected[2].c = "pearl" + local options = {loadFromString=true, headers=false, rename={"a","b","c"}} + local actual = ftcsv.parse("apple>banana>\ndiamond>emerald>pearl", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle only renaming fields from files without headers and only keeping a few fields", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + local options = {loadFromString=true, headers=false, rename={"a","b","c"}, fieldsToKeep={"a","b"}} + local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle only renaming fields from files without headers and only keeping a few fields with an empty header field", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + local options = {loadFromString=true, headers=false, rename={"a","b","c"}, fieldsToKeep={"a","b"}} + local actual = ftcsv.parse("apple>>carrot\ndiamond>emerald>pearl", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle if the number of renames doesn't equal the number of fields", function() + local expected = {} + expected[1] = {} + expected[1].a = "apple" + expected[1].b = "banana" + expected[2] = {} + expected[2].a = "diamond" + expected[2].b = "emerald" + local options = {loadFromString=true, headers=false, rename={"a","b"}, fieldsToKeep={"a","b"}} + local actual = ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", ">", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should make things uppercase via headerFunc", function() + local expected = {} + expected[1] = {} + expected[1].A = "apple" + expected[1].B = "banana" + expected[1].C = "carrot" + local actual = ftcsv.parse("a,b,c\napple,banana,carrot", ",", {loadFromString=true, headerFunc=string.upper}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle encoding files", function() + local expected = {} + expected[1] = {} + expected[1].A = "apple" + expected[1].B = "banana" + expected[1].C = "carrot" + local actual = ftcsv.parse(ftcsv.encode(expected, ","), ",", {loadFromString=true}) + local expected = ftcsv.parse("A,B,C\napple,banana,carrot", ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle encoding files with odd delimiters", function() + local expected = {} + expected[1] = {} + expected[1].A = "apple" + expected[1].B = "banana" + expected[1].C = "carrot" + local actual = ftcsv.parse(ftcsv.encode(expected, ">"), ">", {loadFromString=true}) + local expected = ftcsv.parse("A,B,C\napple,banana,carrot", ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle encoding files with only certain fields to keep", function() + local expected = {} + expected[1] = {} + expected[1].A = "apple" + expected[1].B = "banana" + expected[1].C = "carrot" + local actual = ftcsv.parse(ftcsv.encode(expected, ",", {fieldsToKeep={"A", "B"}}), ",", {loadFromString=true}) + local expected = ftcsv.parse("A,B\napple,banana", ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle encoding files (str test)", function() + local expected = '"a","b","c","d"\r\n"1","","foo","""quoted"""\r\n' + local output = ftcsv.encode({ + { a = 1, b = '', c = 'foo', d = '"quoted"' }; + }, ',') + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files (str test) with other delimiter", function() + local expected = '"a">"b">"c">"d"\r\n"1">"">"foo">"""quoted"""\r\n' + local output = ftcsv.encode({ + { a = 1, b = '', c = 'foo', d = '"quoted"' }; + }, '>') + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files without quotes (str test)", function() + local expected = 'a,b,c,d\r\n1,,"fo,o","""quoted"""\r\n' + local output = ftcsv.encode({ + { a = 1, b = '', c = 'fo,o', d = '"quoted"' }; + }, ',', {onlyRequiredQuotes=true}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files without quotes with other delimiter (str test)", function() + local expected = 'a>b>c>d\r\n1>>fo,o>"""quoted"""\r\n' + local output = ftcsv.encode({ + { a = 1, b = '', c = 'fo,o', d = '"quoted"' }; + }, '>', {onlyRequiredQuotes=true}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files without quotes with certain fields to keep (str test)", function() + local expected = "b,c\r\n,foo\r\n" + local output = ftcsv.encode({ + { a = 1, b = '', c = 'foo', d = '"quoted"' }; + }, ',', {onlyRequiredQuotes=true, fieldsToKeep={"b", "c"}}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files without nil conversion", function() + local expected = '"f1","f2","f3"\r\n"a","b","c"\r\n"d","e","nil"\r\n"nil","nil","f"\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files with nil conversion to empty string", function() + local expected = '"f1","f2","f3"\r\n"a","b","c"\r\n"d","e",""\r\n"","","f"\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=""}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files with nil conversion to number", function() + local expected = '"f1","f2","f3"\r\n"a","b","c"\r\n"d","e","0"\r\n"0","0","f"\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=0}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files with nil conversion to number while only quoting required field", function() + local expected = 'f1,f2,f3\r\na,b,c\r\nd,e,0\r\n0,0,f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=0, onlyRequiredQuotes=true}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files with nil conversion to non-specified delimiter while only quoting required field", function() + local expected = 'f1,f2,f3\r\na,b,c\r\nd,e,","\r\n",",",",f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=",", onlyRequiredQuotes=true}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files with nil conversion to specified delimiter while only quoting required field", function() + local expected = 'f1|f2|f3\r\na|b|c\r\nd|e|,\r\n,|,|f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs=",", onlyRequiredQuotes=true, delimiter="|"}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle encoding files to delimiter with nil conversion to specified delimiter while only quoting required field", function() + local expected = 'f1|f2|f3\r\na|b|c\r\nd|e|"|"\r\n"|"|"|"|f\r\n' + local output = ftcsv.encode({ + {f1 = "a", f2 = "b", f3 = "c"}, + {f1 = "d", f2 = "e",}, + {f3 = "f"}, + }, {encodeNilAs="|", onlyRequiredQuotes=true, delimiter="|"}) + tested.assert({expected=expected, actual=output}) +end) + +tested.test("should handle headers attempting to escape", function() + local expected = {} + expected[1] = {} + expected[1]["]] print('hello')"] = "apple" + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse("]] print('hello'),b,c\napple,banana,carrot", ",", {loadFromString=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle ignoring the single quote", function() + local expected = {} + expected[1] = {} + expected[1].a = '"apple' + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', ",", {loadFromString=true, ignoreQuotes=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle ignoring the single quote without specifying the delimiter", function() + local expected = {} + expected[1] = {} + expected[1].a = '"apple' + expected[1].b = "banana" + expected[1].c = "carrot" + local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', {loadFromString=true, ignoreQuotes=true}) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle reusing the options", function() + local expected = {} + expected[1] = {} + expected[1].a = '"apple' + expected[1].b = "banana" + expected[1].c = "carrot" + local options = {loadFromString=true, ignoreQuotes=true} + local first = ftcsv.parse('a,b,c\n"apple,banana,carrot', ",", options) + local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', ",", options) + tested.assert({expected=expected, actual=actual}) +end) + +tested.test("should handle reusing the options without specifying the delimiter", function() + local expected = {} + expected[1] = {} + expected[1].a = '"apple' + expected[1].b = "banana" + expected[1].c = "carrot" + local options = {loadFromString=true, ignoreQuotes=true} + local first = ftcsv.parse('a,b,c\n"apple,banana,carrot', options) + local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', options) + tested.assert({expected=expected, actual=actual}) +end) + + +return tested \ No newline at end of file diff --git a/tests/json/bom-os9.json b/tests/json/bom-os9.json new file mode 100644 index 0000000..8ced204 --- /dev/null +++ b/tests/json/bom-os9.json @@ -0,0 +1,12 @@ +[ + { + "a": "1", + "b": "2", + "c": "3" + }, + { + "a": "4", + "b": "5", + "c": "ʤ" + } +] \ No newline at end of file diff --git a/tests/json/comma_in_quotes.json b/tests/json/comma_in_quotes.json new file mode 100644 index 0000000..91f3297 --- /dev/null +++ b/tests/json/comma_in_quotes.json @@ -0,0 +1,9 @@ +[ + { + "first": "John", + "last": "Doe", + "address": "120 any st.", + "city": "Anytown, WW", + "zip": "08123" + } +] \ No newline at end of file diff --git a/tests/json/correctness.json b/tests/json/correctness.json new file mode 100644 index 0000000..bc4aa8f --- /dev/null +++ b/tests/json/correctness.json @@ -0,0 +1,37 @@ +[ + { + "Make": "Ford", + "Year": "1997", + "Price": "3000.00", + "Model": "E350", + "Description": "ac, abs, moon" + }, + { + "Make": "Chevy", + "Year": "1999", + "Price": "4900.00", + "Model": "Venture \"Extended Edition\"", + "Description": "" + }, + { + "Make": "Jeep", + "Year": "1996", + "Price": "4799.00", + "Model": "Grand Cherokee", + "Description": "MUST SELL!\nair, moon roof, loaded" + }, + { + "Make": "Chevy", + "Year": "1999", + "Price": "5000.00", + "Model": "Venture \"Extended Edition, Very Large\"", + "Description": "" + }, + { + "Make": "", + "Year": "", + "Price": "4900.00", + "Model": "Venture \"Extended Edition\"", + "Description": "" + } +] \ No newline at end of file diff --git a/tests/json/empty.json b/tests/json/empty.json new file mode 100644 index 0000000..c9a079d --- /dev/null +++ b/tests/json/empty.json @@ -0,0 +1,4 @@ +[ + { "a": "1", "b": "", "c": "" }, + { "a": "2", "b": "3", "c": "4" } +] \ No newline at end of file diff --git a/tests/json/empty_crlf.json b/tests/json/empty_crlf.json new file mode 100644 index 0000000..c9a079d --- /dev/null +++ b/tests/json/empty_crlf.json @@ -0,0 +1,4 @@ +[ + { "a": "1", "b": "", "c": "" }, + { "a": "2", "b": "3", "c": "4" } +] \ No newline at end of file diff --git a/tests/json/empty_no_newline.json b/tests/json/empty_no_newline.json new file mode 100644 index 0000000..cbdf2ec --- /dev/null +++ b/tests/json/empty_no_newline.json @@ -0,0 +1,3 @@ +[ + { "a": "1", "b": "", "c": "" } +] \ No newline at end of file diff --git a/tests/json/empty_no_quotes.json b/tests/json/empty_no_quotes.json new file mode 100644 index 0000000..cbdf2ec --- /dev/null +++ b/tests/json/empty_no_quotes.json @@ -0,0 +1,3 @@ +[ + { "a": "1", "b": "", "c": "" } +] \ No newline at end of file diff --git a/tests/json/escaped_quotes.json b/tests/json/escaped_quotes.json new file mode 100644 index 0000000..bfc19b2 --- /dev/null +++ b/tests/json/escaped_quotes.json @@ -0,0 +1,10 @@ +[ + { + "a": "1", + "b": "ha \"ha\" ha" + }, + { + "a": "3", + "b": "4" + } +] \ No newline at end of file diff --git a/tests/json/escaped_quotes_in_header.json b/tests/json/escaped_quotes_in_header.json new file mode 100644 index 0000000..166ccb7 --- /dev/null +++ b/tests/json/escaped_quotes_in_header.json @@ -0,0 +1,12 @@ +[ + { + "li\"on": "1", + "tiger": "2", + "be\"ar": "3" + }, + { + "li\"on": "5", + "tiger": "6", + "be\"ar": "7" + } +] diff --git a/tests/json/json.json b/tests/json/json.json new file mode 100644 index 0000000..72f21d2 --- /dev/null +++ b/tests/json/json.json @@ -0,0 +1,6 @@ +[ + { + "key": "1", + "val": "{\"type\": \"Point\", \"coordinates\": [102.0, 0.5]}" + } +] \ No newline at end of file diff --git a/tests/json/json_no_newline.json b/tests/json/json_no_newline.json new file mode 100644 index 0000000..72f21d2 --- /dev/null +++ b/tests/json/json_no_newline.json @@ -0,0 +1,6 @@ +[ + { + "key": "1", + "val": "{\"type\": \"Point\", \"coordinates\": [102.0, 0.5]}" + } +] \ No newline at end of file diff --git a/tests/json/missing_keys.json b/tests/json/missing_keys.json new file mode 100644 index 0000000..d22ad49 --- /dev/null +++ b/tests/json/missing_keys.json @@ -0,0 +1,22 @@ +[ + { + "a": "1", + "b": "2", + "c": "3", + "d": "nil" + }, + { + "a": "10", + "b": "20", + "c": "nil", + "d": "nil" + + }, + { + "a": "100", + "b": "nil", + "c": "300", + "d": "nil" + + } +] diff --git a/tests/json/newlines.json b/tests/json/newlines.json new file mode 100644 index 0000000..cb71f1a --- /dev/null +++ b/tests/json/newlines.json @@ -0,0 +1,17 @@ +[ + { + "a": "1", + "b": "2", + "c": "3" + }, + { + "a": "Once upon \na time", + "b": "5", + "c": "6" + }, + { + "a": "7", + "b": "8", + "c": "9" + } +] diff --git a/tests/json/newlines_crlf.json b/tests/json/newlines_crlf.json new file mode 100644 index 0000000..bf13113 --- /dev/null +++ b/tests/json/newlines_crlf.json @@ -0,0 +1,17 @@ +[ + { + "a": "1", + "b": "2", + "c": "3" + }, + { + "a": "Once upon \r\na time", + "b": "5", + "c": "6" + }, + { + "a": "7", + "b": "8", + "c": "9" + } +] diff --git a/tests/json/os9.json b/tests/json/os9.json new file mode 100644 index 0000000..8ced204 --- /dev/null +++ b/tests/json/os9.json @@ -0,0 +1,12 @@ +[ + { + "a": "1", + "b": "2", + "c": "3" + }, + { + "a": "4", + "b": "5", + "c": "ʤ" + } +] \ No newline at end of file diff --git a/tests/json/quotes_and_newlines.json b/tests/json/quotes_and_newlines.json new file mode 100644 index 0000000..160bd3a --- /dev/null +++ b/tests/json/quotes_and_newlines.json @@ -0,0 +1,10 @@ +[ + { + "a": "1", + "b": "ha \n\"ha\" \nha" + }, + { + "a": "3", + "b": "4" + } +] \ No newline at end of file diff --git a/tests/json/quotes_non_escaped.json b/tests/json/quotes_non_escaped.json new file mode 100644 index 0000000..5003f0d --- /dev/null +++ b/tests/json/quotes_non_escaped.json @@ -0,0 +1,8 @@ +[ + { + "Country": "af", + "City": "dekh\"iykh'ya", + "AccentCity": "Dekh\"iykh'ya", + "Region": "13" + } +] diff --git a/tests/json/simple.json b/tests/json/simple.json new file mode 100644 index 0000000..e44a0f7 --- /dev/null +++ b/tests/json/simple.json @@ -0,0 +1,7 @@ +[ + { + "a": "1", + "b": "2", + "c": "3" + } +] diff --git a/tests/json/simple_crlf.json b/tests/json/simple_crlf.json new file mode 100644 index 0000000..1381377 --- /dev/null +++ b/tests/json/simple_crlf.json @@ -0,0 +1,7 @@ +[ + { + "a": "1", + "b": "2", + "c": "3" + } +] \ No newline at end of file diff --git a/tests/json/utf8.json b/tests/json/utf8.json new file mode 100644 index 0000000..8ced204 --- /dev/null +++ b/tests/json/utf8.json @@ -0,0 +1,12 @@ +[ + { + "a": "1", + "b": "2", + "c": "3" + }, + { + "a": "4", + "b": "5", + "c": "ʤ" + } +] \ No newline at end of file diff --git a/tests/parseLine_test.lua b/tests/parseLine_test.lua new file mode 100644 index 0000000..68ce7d5 --- /dev/null +++ b/tests/parseLine_test.lua @@ -0,0 +1,196 @@ +local ftcsv = require('ftcsv') +local cjson = require('cjson') +local tested = require("tested") + +local function loadFile(textFile) + local file = io.open(textFile, "r") + if not file then error("File not found at " .. textFile) end + local allLines = file:read("*all") + file:close() + return allLines +end + +tested.test("parseLine features small, working buffer size", function() + local json = loadFile("spec/json/correctness.json") + json = cjson.decode(json) + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", ",", {bufferSize=52}) do + parse[i] = line + end + tested.assert({ + given="spec/json/correctness.json", + should="handle correctness", + expected=json, + actual=parse + }) +end) + +tested.test("parseLine features small, nonworking buffer size", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", ",", {bufferSize=63}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("parseLine features smaller, nonworking buffer size", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", ",", {bufferSize=50}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("smaller bufferSize than header and incorrect number of fields", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", ",", {bufferSize=23}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("smaller bufferSize than header, but with correct field numbers", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", ",", {bufferSize=30}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("parseLine with options but not bufferSize", function() + + local json = loadFile("spec/json/correctness.json") + json = cjson.decode(json) + + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", ",", {rename={["Year"] = "Full Year"}}) do + parse[i] = line + end + tested.assert({ + given="spec/csvs/correctness.csv", + should="be the same size, even though renamed", + expected=#json, + actual=#parse + }) +end) + +tested.test("parseLine features small, working buffer size without delimiter", function() + local json = loadFile("spec/json/correctness.json") + json = cjson.decode(json) + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", {bufferSize=52}) do + parse[i] = line + end + tested.assert({ + given="spec/csvs/correctness.csv", + expected=json, + actual=parse + }) +end) + +tested.test("parseLine features small, nonworking buffer size without delimiter", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", {bufferSize=63}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("parseLine features smaller, nonworking buffer size without delimiter", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", {bufferSize=50}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("smaller bufferSize than header and incorrect number of fields without delimiter", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", {bufferSize=23}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("smaller bufferSize than header, but with correct field numbers without delimiter", function() + local test = function() + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", {bufferSize=30}) do + parse[i] = line + end + return parse + end + tested.assert_throws_exception({ + given="nonworking buffersize", + expected="ftcsv: bufferSize needs to be larger to parse this file", + actual=test + }) +end) + +tested.test("parseLine with options but not bufferSize without delimiter", function() + local json = loadFile("spec/json/correctness.json") + json = cjson.decode(json) + + local parse = {} + for i, line in ftcsv.parseLine("spec/csvs/correctness.csv", {rename={["Year"] = "Full Year"}}) do + parse[i] = line + end + tested.assert({ + given="spec/csvs/correctness.csv", + should="be the same size, even though renamed", + expected=#json, + actual=#parse + }) + +end) + +return tested \ No newline at end of file diff --git a/tests/parse_encode_test.lua b/tests/parse_encode_test.lua new file mode 100644 index 0000000..8c2d800 --- /dev/null +++ b/tests/parse_encode_test.lua @@ -0,0 +1,154 @@ +local cjson = require("cjson") +local ftcsv = require('ftcsv') +local tested = require("tested") + +local function loadFile(textFile) + local file = io.open(textFile, "r") + if not file then error("File not found at " .. textFile) end + local allLines = file:read("*all") + file:close() + return allLines +end + +local files = { + "bom-os9", + "comma_in_quotes", + "correctness", + "empty", + "empty_no_newline", + "empty_no_quotes", + "empty_crlf", + "escaped_quotes", + "escaped_quotes_in_header", + "json", + "json_no_newline", + "newlines", + "newlines_crlf", + "os9", + "quotes_and_newlines", + "quotes_non_escaped", + "simple", + "simple_crlf", + "utf8" +} + +tested.test("csv decode", function() + for _, value in ipairs(files) do + local json = loadFile("spec/json/" .. value .. ".json") + json = cjson.decode(json) + local parse = ftcsv.parse("spec/csvs/" .. value .. ".csv", ",") + tested.assert({ + given="spec/csvs/" .. value .. ".csv", + should="handle " .. value, + expected=json, + actual=parse + }) + end +end) + +tested.test("csv parseLine decode", function() + for _, value in ipairs(files) do + local json = loadFile("spec/json/" .. value .. ".json") + json = cjson.decode(json) + local parse = {} + for i, v in ftcsv.parseLine("spec/csvs/" .. value .. ".csv", ",") do + parse[i] = v + end + tested.assert({ + given="spec/csvs/" .. value .. ".csv", + should="handle " .. value, + expected = json, + actual=parse + }) + end +end) + +tested.test("csv decode from string", function() + for _, value in ipairs(files) do + local contents = loadFile("spec/csvs/" .. value .. ".csv") + local json = loadFile("spec/json/" .. value .. ".json") + json = cjson.decode(json) + local parse = ftcsv.parse(contents, ",", {loadFromString=true}) + tested.assert({ + given="spec/csvs/" .. value .. ".csv", + should="handle " .. value, + expected = json, + actual=parse + }) + end +end) + +tested.test("csv reencode", function() + for _, value in ipairs(files) do + local jsonFile = loadFile("spec/json/" .. value .. ".json") + local jsonDecode = cjson.decode(jsonFile) + local reEncoded = ftcsv.parse(ftcsv.encode(jsonDecode, ","), ",", {loadFromString=true}) + tested.assert({ + given="spec/json/" .. value .. ".json", + should="handle " .. value, + expected = jsonDecode, + actual=reEncoded + }) + end +end) + +tested.test("csv encode without a delimiter", function() + for _, value in ipairs(files) do + local jsonFile = loadFile("spec/json/" .. value .. ".json") + local jsonDecode = cjson.decode(jsonFile) + local reEncoded = ftcsv.parse(ftcsv.encode(jsonDecode), ",", {loadFromString=true}) + tested.assert({ + given="spec/json/" .. value .. ".json", + should="handle " .. value, + expected = jsonDecode, + actual=reEncoded + }) + end +end) + +tested.test("csv encode with a delimiter specified in options", function() + for _, value in ipairs(files) do + local jsonFile = loadFile("spec/json/" .. value .. ".json") + local jsonDecode = cjson.decode(jsonFile) + local reEncoded = ftcsv.parse(ftcsv.encode(jsonDecode, {delimiter="\t"}), {delimiter="\t", loadFromString=true}) + tested.assert({ + given="spec/json/" .. value .. ".json", + should="handle " .. value, + expected = jsonDecode, + actual=reEncoded + }) + end +end) + +tested.test("csv encode without quotes", function() + for _, value in ipairs(files) do + local jsonFile = loadFile("spec/json/" .. value .. ".json") + local jsonDecode = cjson.decode(jsonFile) + local reEncodedNoQuotes = ftcsv.parse(ftcsv.encode(jsonDecode, ",", {onlyRequiredQuotes=true}), ",", {loadFromString=true}) + tested.assert({ + given="spec/json/" .. value .. ".json", + should="handle " .. value, + expected = jsonDecode, + actual=reEncodedNoQuotes + }) + end +end) + +tested.test("csv encode with missing keys", function() + local jsonFile = loadFile("spec/json/missing_keys.json") + local jsonDecode = cjson.decode(jsonFile) + local reEncoded = ftcsv.parse(ftcsv.encode( + jsonDecode, ",", { + fieldsToKeep = {"a", "b", "c", "d"}, + allowMissingKeys = true, + } + ), ",", {loadFromString=true}) + tested.assert({ + given="spec/json/missing_keys.json", + should="handle missing_keys", + expected = jsonDecode, + actual=reEncoded + }) +end) + +return tested \ No newline at end of file