add tests for cleanup on target side

Author: majewsky

pkg/actors/cleaner.go

@@ -80,10 +80,13 @@ func (c *Cleaner) Run() {
 
 	//perform cleanup if it is safe to do so
 	for job, transferred := range isFileTransferred {
-		if isJobFailed[job] || c.Context.Err() == nil {
-			continue
+		if c.Context.Err() != nil {
+			//interrupt received
+			return
+		}
+		if !isJobFailed[job] {
+			c.performCleanup(job, transferred)
 		}
-		c.performCleanup(job, transferred)
 	}
 }
 

pkg/objects/swift.go

@@ -265,11 +265,7 @@ func (s *SwiftLocation) DiscoverExistingFiles(matcher Matcher) error {
 	iter.Prefix = prefix
 	s.FileExists = make(map[string]bool)
 	err := iter.Foreach(func(object *schwift.Object) error {
-		path := object.Name()
-		pathForMatching := strings.TrimPrefix(path, prefix)
-		if matcher.CheckRecursive(pathForMatching) == "" {
-			s.FileExists[path] = true
-		}
+		s.FileExists[object.Name()] = true
 		return nil
 	})
 	if err != nil {

tests.sh

@@ -89,6 +89,16 @@ fi
 ################################################################################
 # functions for tests
 
+upload_target_file_from_stdin() {
+  # same trickery as in `upload_file_from_stdin` (see comments over there), but
+  # for a target container
+  sed 's/^  //' > "${TEST_FILENAME}"
+  local CONTAINER_NAME="${CONTAINER_BASE}-$1"
+  local OBJECT_NAME="$2"
+  shift 2
+  swift upload "${CONTAINER_NAME}" "${TEST_FILENAME}" --object-name "${OBJECT_NAME}" "$@"
+}
+
 mirror() {
   # config file comes from stdin
   ./build/swift-http-import /dev/fd/0
@@ -494,6 +504,64 @@ fi
 
 fi # end of: if [ "$1" = http ]
 
+################################################################################
+step 'Test 10: Cleanup on target side'
+
+upload_target_file_from_stdin test10 ignored.txt <<-EOF
+  This file does not get cleaned up because it's not below object_prefix.
+EOF
+upload_target_file_from_stdin test10 target/cleanup-please.txt <<-EOF
+  This file gets cleaned up because it's below object_prefix.
+EOF
+# This file will get cleaned up even though it exists on the source side
+# because it's excluded from transfer by a filter.
+upload_target_file_from_stdin test10 target/just/some/files/2.txt <<-EOF
+  Hello Second World.
+EOF
+sleep 10 # wait for container listing to get updated
+
+mirror <<-EOF
+  swift: { $AUTH_PARAMS }
+  jobs:
+    - from: ${SOURCE_SPEC}
+      to:
+        container: ${CONTAINER_BASE}-test10
+        object_prefix: target/
+      only: '/$|1\\.txt$'
+EOF
+
+# This first pass does not have cleanup enabled (to compare against the second
+# pass down below), so we're still seeing the files that need to be cleaned up.
+expect test10 <<-EOF
+>> ignored.txt
+This file does not get cleaned up because it's not below object_prefix.
+>> target/cleanup-please.txt
+This file gets cleaned up because it's below object_prefix.
+>> target/just/some/files/1.txt
+Hello World.
+>> target/just/some/files/2.txt
+Hello Second World.
+EOF
+
+mirror <<-EOF
+  swift: { $AUTH_PARAMS }
+  jobs:
+    - from: ${SOURCE_SPEC}
+      to:
+        container: ${CONTAINER_BASE}-test10
+        object_prefix: target/
+      only: '/$|1\\.txt$'
+      cleanup:
+        strategy: delete
+EOF
+
+expect test10 <<-EOF
+>> ignored.txt
+This file does not get cleaned up because it's not below object_prefix.
+>> target/just/some/files/1.txt
+Hello World.
+EOF
+
 ################################################################################
 # cleanup before exiting