fix: add remove event update index num

wantWhatBike · wantWhatBike · commit 1d9970702b6d · 2025-08-21T16:48:34.000+08:00
diff --git a/internal/service/indexer.go b/internal/service/indexer.go
@@ -399,37 +399,47 @@ func (i *indexer) RemoveIndexes(ctx context.Context, workspacePath string, fileP
 	if len(projects) == 0 {
 		return fmt.Errorf("no project found in workspace %s", workspacePath)
 	}
-
+	workspaceModel, err := i.workspaceRepository.GetWorkspaceByPath(workspacePath)
+	if err != nil {
+		return err
+	}
 	var errs []error
 	projectFilesMap, err := i.groupFilesByProject(projects, filePaths)
 	if err != nil {
 		return fmt.Errorf("group files by project failed: %w", err)
 	}
-
+	totalRemoved := 0
 	for projectUuid, files := range projectFilesMap {
 		pStart := time.Now()
 		i.logger.Info("start to remove project %s files index", projectUuid)
 
-		if err := i.removeIndexByFilePaths(ctx, projectUuid, files); err != nil {
+		removed, err := i.removeIndexByFilePaths(ctx, projectUuid, files)
+		if err != nil {
 			errs = append(errs, err)
 		}
-
-		i.logger.Info("remove project %s files index end, cost %d ms", projectUuid,
-			time.Since(pStart).Milliseconds())
+		totalRemoved += removed
+		i.logger.Info("remove project %s files index end, cost %d ms, removed %d index.", projectUuid,
+			time.Since(pStart).Milliseconds(), removed)
+	}
+	// 更新为删除后的值
+	if workspaceModel != nil {
+		if err := i.workspaceRepository.UpdateCodegraphInfo(workspacePath,
+			workspaceModel.CodegraphFileNum-totalRemoved, time.Now().Unix()); err != nil {
+			return errors.Join(append(errs, err)...)
+		}
 	}
-
 	err = errors.Join(errs...)
-	i.logger.Info("remove workspace %s files index successfully, cost %d ms, errors: %v",
-		workspacePath, time.Since(start).Milliseconds(), utils.TruncateError(err))
+	i.logger.Info("remove workspace %s files index successfully, cost %d ms, removed %d index, errors: %v",
+		workspacePath, time.Since(start).Milliseconds(), totalRemoved, utils.TruncateError(err))
 	return err
 }
 
 // removeIndexByFilePaths 删除单个项目的索引
-func (i *indexer) removeIndexByFilePaths(ctx context.Context, projectUuid string, filePaths []string) error {
+func (i *indexer) removeIndexByFilePaths(ctx context.Context, projectUuid string, filePaths []string) (int, error) {
 	// 1. 查询path相应的file_table
 	deleteFileTables, err := i.searchFileElementTablesByPath(ctx, projectUuid, filePaths)
 	if err != nil {
-		return fmt.Errorf("get file tables for deletion failed: %w", err)
+		return 0, fmt.Errorf("get file tables for deletion failed: %w", err)
 	}
 	deletePaths := make(map[string]any)
 	for _, v := range deleteFileTables {
@@ -438,15 +448,16 @@ func (i *indexer) removeIndexByFilePaths(ctx context.Context, projectUuid string
 
 	// 2. 清理符号定义
 	if err = i.cleanupSymbolOccurrences(ctx, projectUuid, deleteFileTables, deletePaths); err != nil {
-		return fmt.Errorf("cleanup symbol definitions failed: %w", err)
+		return 0, fmt.Errorf("cleanup symbol definitions failed: %w", err)
 	}
 
 	// 3. 删除path索引
-	if err = i.deleteFileIndexes(ctx, projectUuid, filePaths); err != nil {
-		return fmt.Errorf("delete file indexes failed: %w", err)
+	deleted, err := i.deleteFileIndexes(ctx, projectUuid, filePaths)
+	if err != nil {
+		return 0, fmt.Errorf("delete file indexes failed: %w", err)
 	}
 
-	return nil
+	return deleted, nil
 }
 
 // searchFileElementTablesByPath 获取待删除的文件表和路径（包括文件夹）
@@ -572,9 +583,9 @@ func (i *indexer) cleanupSymbolOccurrences(ctx context.Context, projectUuid stri
 }
 
 // deleteFileIndexes 删除文件索引
-func (i *indexer) deleteFileIndexes(ctx context.Context, puuid string, filePaths []string) error {
+func (i *indexer) deleteFileIndexes(ctx context.Context, puuid string, filePaths []string) (int, error) {
 	var errs []error
-
+	deleted := 0
 	for _, fp := range filePaths {
 		// 删除path索引
 		language, err := lang.InferLanguage(fp)
@@ -583,14 +594,16 @@ func (i *indexer) deleteFileIndexes(ctx context.Context, puuid string, filePaths
 		}
 		if err = i.storage.Delete(ctx, puuid, store.ElementPathKey{Language: language, Path: fp}); err != nil {
 			errs = append(errs, err)
+		} else {
+			deleted++
 		}
 	}
 
 	if len(errs) > 0 {
-		return errors.Join(errs...)
+		return deleted, errors.Join(errs...)
 	}
 
-	return nil
+	return deleted, nil
 }
 
 // IndexFiles 根据工作区路径、文件路径，批量保存索引
@@ -652,7 +665,7 @@ func (i *indexer) IndexFiles(ctx context.Context, workspacePath string, filePath
 				TotalFilesCnt:        len(projectFiles),
 				Project:              project,
 				WorkspacePath:        workspacePath,
-				PreviousFileNum:      workspaceModel.FileNum - len(projectFiles),
+				PreviousFileNum:      workspaceModel.FileNum,
 				Concurrency:          i.config.MaxConcurrency,
 				BatchSize:            i.config.MaxBatchSize,
 			}
@@ -1424,6 +1437,10 @@ func (i *indexer) RemoveAllIndexes(ctx context.Context, workspacePath string) er
 	for _, p := range projects {
 		errs = append(errs, i.storage.DeleteAll(ctx, p.Uuid))
 	}
+	// 将数据库数据置为0
+	if err := i.workspaceRepository.UpdateCodegraphInfo(workspacePath, 0, time.Now().Unix()); err != nil {
+		return errors.Join(append(errs, fmt.Errorf("update codegraph info err:%v", err))...)
+	}
 	return errors.Join(errs...)
 }
 
@@ -1580,7 +1597,6 @@ func (i *indexer) parseFiles(ctx context.Context, files []*types.FileWithModTime
 		FailedFilePaths: make([]string, 0, totalFiles/4), // 预估失败文件数约为文件数的25%
 	}
 
-	// TODO 大量文件，会导致错误很多
 	var errs []error
 
 	for _, f := range files {
diff --git a/test/api/index_by_event_test.go b/test/api/index_by_event_test.go
@@ -0,0 +1,233 @@
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type IndexEventIntegrationTestSuite struct {
+	BaseIntegrationTestSuite
+}
+
+type indexEventTestCase struct {
+	name            string
+	workspace       string
+	data            []map[string]interface{}
+	wantProcessTime time.Duration
+	validateIndex   func(t *testing.T, indexes []map[string]interface{})
+}
+
+func (s *IndexEventIntegrationTestSuite) TestPublishEvent() {
+	// 定义测试用例表
+	testCases := []indexEventTestCase{
+		{
+			name:      "打开工作区",
+			workspace: s.workspacePath,
+			data: []map[string]interface{}{
+				{
+					"eventType": "open_workspace",
+					"eventTime": "2025-07-28 20:47:00",
+				},
+			},
+			wantProcessTime: time.Second * 15,
+			validateIndex:   func(t *testing.T, indexes []map[string]interface{}) {},
+		},
+
+		{
+			name: "删除文件",
+			data: []map[string]interface{}{
+				{
+					"eventType":  "delete_file",
+					"eventTime":  "2025-07-28 20:47:00",
+					"sourcePath": s.workspacePath,
+					"targetPath": s.workspacePath,
+				},
+			},
+			wantProcessTime: time.Second * 15,
+			validateIndex:   func(t *testing.T, indexes []map[string]interface{}) {},
+		},
+		{
+			name:      "删除文件夹",
+			workspace: s.workspacePath,
+			data: []map[string]interface{}{
+				{
+					"eventType":  "delete_file",
+					"eventTime":  "2025-07-28 20:47:00",
+					"sourcePath": s.workspacePath,
+					"targetPath": s.workspacePath,
+				},
+			},
+			wantProcessTime: time.Second * 15,
+			validateIndex:   func(t *testing.T, indexes []map[string]interface{}) {},
+		},
+		{
+			name:      "新增文件",
+			workspace: s.workspacePath,
+			data: []map[string]interface{}{
+				{
+					"eventType":  "add_file",
+					"eventTime":  "2025-07-28 20:47:00",
+					"sourcePath": s.workspacePath,
+					"targetPath": s.workspacePath,
+				},
+			},
+			wantProcessTime: time.Second * 15,
+			validateIndex:   func(t *testing.T, indexes []map[string]interface{}) {},
+		},
+		{
+			name:      "修改文件",
+			workspace: s.workspacePath,
+			data: []map[string]interface{}{
+				{
+					"eventType":  "modify_file",
+					"eventTime":  "2025-07-28 20:47:00",
+					"sourcePath": s.workspacePath,
+					"targetPath": s.workspacePath,
+				},
+			},
+			wantProcessTime: time.Second * 15,
+			validateIndex:   func(t *testing.T, indexes []map[string]interface{}) {},
+		},
+		{
+			name:      "重命名文件",
+			workspace: s.workspacePath,
+			data: []map[string]interface{}{
+				{
+					"eventType":  "rename_file",
+					"eventTime":  "2025-07-28 20:47:00",
+					"sourcePath": s.workspacePath,
+					"targetPath": s.workspacePath,
+				},
+			},
+			wantProcessTime: time.Second * 15,
+			validateIndex:   func(t *testing.T, indexes []map[string]interface{}) {},
+		},
+		{
+			name:      "重命名文件夹",
+			workspace: s.workspacePath,
+			data: []map[string]interface{}{
+				{
+					"eventType":  "rename_file",
+					"eventTime":  "2025-07-28 20:47:00",
+					"sourcePath": s.workspacePath,
+					"targetPath": s.workspacePath,
+				},
+			},
+			wantProcessTime: time.Second * 15,
+			validateIndex:   func(t *testing.T, indexes []map[string]interface{}) {},
+		},
+	}
+
+	// 执行表格驱动测试
+	for _, tc := range testCases {
+		s.T().Run(tc.name, func(t *testing.T) {
+			// 发布事件
+
+			// 等待一定事件
+
+			// 查询索引，达到期望状态
+
+		})
+	}
+}
+
+func TestIndexEventIntegrationTestSuite(t *testing.T) {
+	suite.Run(t, new(IndexEventIntegrationTestSuite))
+}
+
+// publishEvent 发布索引事件
+func (s *IndexEventIntegrationTestSuite) publishEvent(workspace string, data any) error {
+	var resp *http.Response
+	var err error
+
+	// 准备请求体
+	reqBody := map[string]interface{}{
+		"workspace": workspace,
+		"data":      data,
+	}
+
+	// 序列化请求体
+	jsonData, err := json.Marshal(reqBody)
+	if err != nil {
+		return err
+	}
+
+	// 创建HTTP请求
+	req, err := s.CreatePOSTRequest(s.baseURL+"/codebase-indexer/api/v1/events", jsonData)
+	if err != nil {
+		return err
+	}
+
+	// 发送请求
+	resp, err = s.SendRequest(req)
+
+	s.Require().NoError(err)
+	defer resp.Body.Close()
+
+	// 验证响应状态码
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+	// 读取响应体
+	body, err := io.ReadAll(resp.Body)
+	s.Require().NoError(err)
+
+	// 解析响应JSON
+	var response map[string]interface{}
+	if err = json.Unmarshal(body, &response); err != nil {
+		return err
+	}
+
+	s.Equal("ok", response["message"])
+	if response["message"] != "ok" {
+		return fmt.Errorf("expected message `ok` but got `%s`", response["message"])
+	}
+	return nil
+}
+
+// 获取全部索引
+func (s *IndexEventIntegrationTestSuite) dumpIndex() ([]map[string]interface{}, error) {
+	// 构建请求URL
+	reqURL, err := url.Parse(s.baseURL + "/codebase-indexer/api/v1/index/export")
+	s.Require().NoError(err)
+
+	// 添加查询参数
+	q := reqURL.Query()
+	q.Add("codebasePath", s.workspacePath)
+
+	reqURL.RawQuery = q.Encode()
+
+	// 创建HTTP请求
+	req, err := s.CreateGETRequest(reqURL.String())
+	s.Require().NoError(err)
+
+	// 发送请求
+	resp, err := s.SendRequest(req)
+	s.Require().NoError(err)
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+
+	// 先将读取到的body转换为字符串
+	bodyStr := string(body)
+
+	// 按行分割内容
+	lines := strings.Split(bodyStr, "\n")
+	var indexes []map[string]interface{}
+	for _, line := range lines {
+		var indexLine map[string]interface{}
+		err := json.Unmarshal([]byte(line), &indexLine)
+		if err != nil {
+			return nil, err
+		}
+		indexes = append(indexes, indexLine)
+	}
+	return indexes, nil
+}
