adjust max number of lines for CSV file imports

Author: whummer

.github/workflows/build_test.yml

@@ -26,6 +26,7 @@ concurrency:
 
 env:
   USE_PASSWORD: "true"
+  SF_CSV_IMPORT_MAX_ROWS: "30000"
 
 jobs:
   lint:
@@ -168,7 +169,7 @@ jobs:
           # image_name=whummer/snowflake-tmp
           docker pull $image_name &
           pip install localstack
-          IMAGE_NAME=$image_name DEBUG=1 DOCKER_FLAGS='-e SF_LOG=trace' localstack start -d
+          IMAGE_NAME=$image_name DEBUG=1 DOCKER_FLAGS='-e SF_LOG=trace -e SF_CSV_IMPORT_MAX_ROWS=30000' localstack start -d
           localstack wait
 
 #      - name: Setup parameters file
@@ -203,7 +204,7 @@ jobs:
          # To specify the test name (in single test mode) pass this env variable:
 #          SINGLE_TEST_NAME: test/path/filename.py::test_name
         shell: bash
-        timeout-minutes: 15
+        timeout-minutes: 20
       - name: Combine coverages
         run: python -m tox run -e coverage --skip-missing-interpreters false
         shell: bash

test/integ/test_large_put.py

@@ -18,6 +18,10 @@ def test_put_copy_large_files(tmpdir, conn_cnx, db_parameters):
     # generates N files
     number_of_files = 2
     number_of_lines = 200000
+
+    # note: adjusted for compatibility with LocalStack emulator tests
+    number_of_lines_expected = int(os.getenv("SF_CSV_IMPORT_MAX_ROWS"))
+
     tmp_dir = generate_k_lines_of_n_files(
         number_of_lines, number_of_files, tmp_dir=str(tmpdir.mkdir("data"))
     )
@@ -91,7 +95,9 @@ def mocked_file_agent(*args, **kwargs):
                 cnt = 0
                 for rec in c:
                     cnt += rec[0]
-                assert cnt == number_of_files * number_of_lines, "Number of rows"
+                assert (
+                    cnt == number_of_files * number_of_lines_expected
+                ), "Number of rows"
             finally:
                 c.close()
     finally:

test/integ/test_put_get_medium.py

@@ -400,9 +400,11 @@ def run(cnx, sql):
 @pytest.mark.flaky(reruns=3)
 def test_put_copy_many_files(tmpdir, conn_cnx, db_parameters):
     """Puts and Copies many_files."""
-    # generates N files
-    number_of_files = 100
-    number_of_lines = 1000
+
+    # generates N files - note: adjusted for LocalStack, to reduce CI test times
+    number_of_files = 50
+    number_of_lines = 500
+
     tmp_dir = generate_k_lines_of_n_files(
         number_of_lines, number_of_files, tmp_dir=str(tmpdir.mkdir("data"))
     )