diff --git a/pkg/sources/s3/s3_integration_test.go b/pkg/sources/s3/s3_integration_test.go index 3276e40fd..02eea5611 100644 --- a/pkg/sources/s3/s3_integration_test.go +++ b/pkg/sources/s3/s3_integration_test.go @@ -82,37 +82,6 @@ func TestSource_ChunksLarge(t *testing.T) { assert.Equal(t, got, wantChunkCount) } -func TestSourceChunksNoResumption(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - - s := Source{} - connection := &sourcespb.S3{ - Credential: &sourcespb.S3_Unauthenticated{}, - Buckets: []string{"trufflesec-ahrav-test-2"}, - } - conn, err := anypb.New(connection) - if err != nil { - t.Fatal(err) - } - - err = s.Init(ctx, "test name", 0, 0, false, conn, 1) - chunksCh := make(chan *sources.Chunk) - go func() { - defer close(chunksCh) - err = s.Chunks(ctx, chunksCh) - assert.Nil(t, err) - }() - - wantChunkCount := 19787 - got := 0 - - for range chunksCh { - got++ - } - assert.Equal(t, got, wantChunkCount) -} - func TestSource_Validate(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer cancel() @@ -251,34 +220,50 @@ func TestSource_Validate(t *testing.T) { func TestSourceChunksNoResumption(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - - s := Source{} - connection := &sourcespb.S3{ - Credential: &sourcespb.S3_Unauthenticated{}, - Buckets: []string{"integration-resumption-tests"}, - } - conn, err := anypb.New(connection) - if err != nil { - t.Fatal(err) + tests := []struct { + bucket string + wantChunkCount int + }{ + { + bucket: "trufflesec-ahrav-test-2", + wantChunkCount: 19787, + }, + { + bucket: "integration-resumption-tests", + wantChunkCount: 19787, + }, } - err = s.Init(ctx, "test name", 0, 0, false, conn, 1) - chunksCh := make(chan *sources.Chunk) - go func() { - defer close(chunksCh) - err = s.Chunks(ctx, chunksCh) - assert.Nil(t, err) - }() + for _, tt := range tests { + t.Run(tt.bucket, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() - wantChunkCount := 19787 - got := 0 + s := Source{} + connection := &sourcespb.S3{ + Credential: &sourcespb.S3_Unauthenticated{}, + Buckets: []string{tt.bucket}, + } + conn, err := anypb.New(connection) + if err != nil { + t.Fatal(err) + } - for range chunksCh { - got++ + err = s.Init(ctx, "test name", 0, 0, false, conn, 1) + chunksCh := make(chan *sources.Chunk) + go func() { + defer close(chunksCh) + err = s.Chunks(ctx, chunksCh) + assert.Nil(t, err) + }() + + got := 0 + for range chunksCh { + got++ + } + assert.Equal(t, tt.wantChunkCount, got) + }) } - assert.Equal(t, wantChunkCount, got) } func TestSourceChunksResumption(t *testing.T) { diff --git a/pkg/sources/s3/s3_test.go b/pkg/sources/s3/s3_test.go index 5f2f4aed7..4f0892ed3 100644 --- a/pkg/sources/s3/s3_test.go +++ b/pkg/sources/s3/s3_test.go @@ -4,7 +4,6 @@ import ( "encoding/base64" "fmt" "os" - "sync" "testing" "time" @@ -99,8 +98,7 @@ func TestSource_Chunks(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - var cancelOnce sync.Once - defer cancelOnce.Do(cancel) + defer cancel() for k, v := range tt.init.setEnv { t.Setenv(k, v) @@ -117,26 +115,47 @@ func TestSource_Chunks(t *testing.T) { t.Errorf("Source.Init() error = %v, wantErr %v", err, tt.wantErr) return } - chunksCh := make(chan *sources.Chunk) - var wg sync.WaitGroup - wg.Add(1) + chunksCh := make(chan *sources.Chunk, 1) go func() { - defer wg.Done() + defer close(chunksCh) err = s.Chunks(ctx, chunksCh) if (err != nil) != tt.wantErr { t.Errorf("Source.Chunks() error = %v, wantErr %v", err, tt.wantErr) os.Exit(1) } }() - gotChunk := <-chunksCh - wantData, _ := base64.StdEncoding.DecodeString(tt.wantChunkData) - if diff := pretty.Compare(gotChunk.Data, wantData); diff != "" { - t.Errorf("%s: Source.Chunks() diff: (-got +want)\n%s", tt.name, diff) + waitFn := func() { + receivedFirstChunk := false + for { + select { + case <-ctx.Done(): + t.Errorf("TestSource_Chunks timed out: %v", ctx.Err()) + return + case gotChunk, ok := <-chunksCh: + if !ok { + t.Logf("Source.Chunks() finished, channel closed") + assert.Equal(t, "", s.GetProgress().EncodedResumeInfo) + assert.Equal(t, int64(100), s.GetProgress().PercentComplete) + return + } + if receivedFirstChunk { + // wantChunkData is the first chunk data. After the first chunk has + // been received and matched below, we want to drain chunksCh + // so Source.Chunks() can finish completely. + continue + } + + receivedFirstChunk = true + wantData, _ := base64.StdEncoding.DecodeString(tt.wantChunkData) + + if diff := pretty.Compare(gotChunk.Data, wantData); diff != "" { + t.Logf("%s: Source.Chunks() diff: (-got +want)\n%s", tt.name, diff) + } + } + } } - wg.Wait() - assert.Equal(t, "", s.GetProgress().EncodedResumeInfo) - assert.Equal(t, int64(100), s.GetProgress().PercentComplete) + waitFn() }) } }