216 search by job name (#294)

* add job filter

* add job filter

* add jobs

* fix rs

* fix branch

* add job filter for latest pipelines

* fix branches

* fix bug

* add to schedules

* fix filter

* only fetch failed jobs

* fix tests

Author: larscom

api/src/branch/pipeline_aggregator.rs

@@ -1,6 +1,7 @@
 use crate::branch::branch_service::BranchService;
 use crate::error::ApiError;
-use crate::model::{Branch, BranchPipeline};
+use crate::job::JobService;
+use crate::model::{Branch, BranchPipeline, JobStatus, PipelineStatus};
 use crate::pipeline;
 use crate::pipeline::PipelineService;
 use crate::util::iter::try_collect_with_buffer;
@@ -9,13 +10,19 @@ use pipeline::sort_by_updated_date;
 pub struct PipelineAggregator {
     branch_service: BranchService,
     pipeline_service: PipelineService,
+    job_service: JobService,
 }
 
 impl PipelineAggregator {
-    pub fn new(branch_service: BranchService, pipeline_service: PipelineService) -> Self {
+    pub fn new(
+        branch_service: BranchService,
+        pipeline_service: PipelineService,
+        job_service: JobService,
+    ) -> Self {
         Self {
             branch_service,
             pipeline_service,
+            job_service,
         }
     }
 }
@@ -45,7 +52,21 @@ impl PipelineAggregator {
                 .pipeline_service
                 .get_latest_pipeline(project_id, branch.name.clone())
                 .await?;
-            Ok(BranchPipeline { branch, pipeline })
+
+            let failed_jobs = match pipeline {
+                Some(ref p) if p.status == PipelineStatus::Failed => Some(
+                    self.job_service
+                        .get_jobs(p.project_id, p.id, &[JobStatus::Failed])
+                        .await?,
+                ),
+                _ => None,
+            };
+
+            Ok(BranchPipeline {
+                branch,
+                pipeline,
+                failed_jobs,
+            })
         })
         .await
     }

api/src/job/job_service.rs

@@ -22,6 +22,7 @@ impl CacheKey {
     }
 }
 
+#[derive(Clone)]
 pub struct JobService {
     cache: Cache<CacheKey, Vec<Job>>,
     client: Arc<dyn GitlabApi>,

api/src/main.rs

@@ -83,15 +83,18 @@ async fn main() -> std::io::Result<()> {
     let project_aggr = Data::new(project::PipelineAggregator::new(
         project_service.get_ref().clone(),
         pipeline_service.get_ref().clone(),
+        job_service.get_ref().clone(),
     ));
     let branch_aggr = Data::new(branch::PipelineAggregator::new(
         branch_service.get_ref().clone(),
         pipeline_service.get_ref().clone(),
+        job_service.get_ref().clone(),
     ));
     let schedule_aggr = Data::new(schedule::PipelineAggregator::new(
         schedule::ScheduleService::new(gitlab_client.clone(), app_config.clone()),
         project_service.get_ref().clone(),
         pipeline_service.get_ref().clone(),
+        job_service.get_ref().clone(),
     ));
 
     let prom = setup_prometheus();
@@ -245,15 +248,18 @@ mod tests {
             let project_aggr = Data::new(project::PipelineAggregator::new(
                 project_service.get_ref().clone(),
                 pipeline_service.get_ref().clone(),
+                job_service.get_ref().clone(),
             ));
             let branch_aggr = Data::new(branch::PipelineAggregator::new(
                 branch_service.get_ref().clone(),
                 pipeline_service.get_ref().clone(),
+                job_service.get_ref().clone(),
             ));
             let schedule_aggr = Data::new(schedule::PipelineAggregator::new(
                 schedule::ScheduleService::new(gitlab_client.clone(), gcd_config.clone()),
                 project_service.get_ref().clone(),
                 pipeline_service.get_ref().clone(),
+                job_service.get_ref().clone(),
             ));
 
             test::init_service(App::new().configure(configure_app(

api/src/model/branch.rs

@@ -1,7 +1,7 @@
 use serde::{Deserialize, Serialize};
 
 use crate::model::commit::Commit;
-use crate::model::Pipeline;
+use crate::model::{Job, Pipeline};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Branch {
@@ -19,13 +19,15 @@ pub struct BranchPipeline {
     pub branch: Branch,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub pipeline: Option<Pipeline>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub failed_jobs: Option<Vec<Job>>,
 }
 
 #[cfg(test)]
 mod tests {
     use serde_json::json;
 
-    use crate::model::{Branch, BranchPipeline, test};
+    use crate::model::{test, Branch, BranchPipeline};
 
     #[test]
     fn branch_deserialize() {
@@ -69,6 +71,7 @@ mod tests {
 
         let json = serde_json::to_string(&value).unwrap();
         let expected = "{\"name\":\"branch-1\",\"merged\":false,\"protected\":false,\"default\":false,\"can_push\":false,\"web_url\":\"web_url\",\"commit\":{\"id\":\"id\",\"author_name\":\"author_name\",\"committer_name\":\"committer_name\",\"committed_date\":\"1970-01-01T00:00:00Z\",\"title\":\"title\",\"message\":\"message\"}}";
+
         assert_eq!(expected, json);
     }
 
@@ -77,10 +80,12 @@ mod tests {
         let value = BranchPipeline {
             branch: test::new_branch(),
             pipeline: None,
+            failed_jobs: None,
         };
 
         let json = serde_json::to_string(&value).unwrap();
         let expected = "{\"branch\":{\"name\":\"branch-1\",\"merged\":false,\"protected\":false,\"default\":false,\"can_push\":false,\"web_url\":\"web_url\",\"commit\":{\"id\":\"id\",\"author_name\":\"author_name\",\"committer_name\":\"committer_name\",\"committed_date\":\"1970-01-01T00:00:00Z\",\"title\":\"title\",\"message\":\"message\"}}}";
+
         assert_eq!(expected, json);
     }
 
@@ -89,10 +94,26 @@ mod tests {
         let value = BranchPipeline {
             branch: test::new_branch(),
             pipeline: Some(test::new_pipeline()),
+            failed_jobs: None,
         };
 
         let json = serde_json::to_string(&value).unwrap();
         let expected = "{\"branch\":{\"name\":\"branch-1\",\"merged\":false,\"protected\":false,\"default\":false,\"can_push\":false,\"web_url\":\"web_url\",\"commit\":{\"id\":\"id\",\"author_name\":\"author_name\",\"committer_name\":\"committer_name\",\"committed_date\":\"1970-01-01T00:00:00Z\",\"title\":\"title\",\"message\":\"message\"}},\"pipeline\":{\"id\":1,\"iid\":2,\"project_id\":3,\"sha\":\"sha\",\"ref\":\"branch\",\"status\":\"running\",\"source\":\"web\",\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"web_url\":\"web_url\"}}";
+
+        assert_eq!(expected, json);
+    }
+
+    #[test]
+    fn branch_pipeline_serialize_some_failed_jobs() {
+        let value = BranchPipeline {
+            branch: test::new_branch(),
+            pipeline: None,
+            failed_jobs: Some(vec![test::new_job()]),
+        };
+
+        let json = serde_json::to_string(&value).unwrap();
+        let expected = "{\"branch\":{\"name\":\"branch-1\",\"merged\":false,\"protected\":false,\"default\":false,\"can_push\":false,\"web_url\":\"web_url\",\"commit\":{\"id\":\"id\",\"author_name\":\"author_name\",\"committer_name\":\"committer_name\",\"committed_date\":\"1970-01-01T00:00:00Z\",\"title\":\"title\",\"message\":\"message\"}},\"failed_jobs\":[{\"id\":1,\"created_at\":\"1970-01-01T00:00:00Z\",\"allow_failure\":false,\"name\":\"name\",\"ref\":\"branch\",\"stage\":\"stage\",\"status\":\"success\",\"web_url\":\"web_url\",\"pipeline\":{\"id\":1,\"iid\":2,\"project_id\":3,\"sha\":\"sha\",\"ref\":\"branch\",\"status\":\"running\",\"source\":\"web\",\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"web_url\":\"web_url\"},\"commit\":{\"id\":\"id\",\"author_name\":\"author_name\",\"committer_name\":\"committer_name\",\"committed_date\":\"1970-01-01T00:00:00Z\",\"title\":\"title\",\"message\":\"message\"},\"user\":{\"id\":123,\"username\":\"username\",\"name\":\"name\",\"state\":\"state\",\"is_admin\":false}}]}";
+
         assert_eq!(expected, json);
     }
 }

api/src/model/pipeline.rs

@@ -16,7 +16,7 @@ pub struct Pipeline {
     pub web_url: String,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
 #[serde(rename_all = "lowercase")]
 pub enum PipelineStatus {
     /// Currently running.

api/src/model/project.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 
-use crate::model::Pipeline;
+use crate::model::{Job, Pipeline};
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Project {
@@ -9,14 +9,14 @@ pub struct Project {
     pub web_url: String,
     pub default_branch: Option<String>,
     pub topics: Vec<String>,
-    pub namespace: Namespace
+    pub namespace: Namespace,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct Namespace {
     pub id: u64,
     pub name: String,
-    pub path: String
+    pub path: String,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -25,6 +25,8 @@ pub struct ProjectPipeline {
     pub project: Project,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub pipeline: Option<Pipeline>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub failed_jobs: Option<Vec<Job>>,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -36,7 +38,7 @@ pub struct ProjectPipelines {
 
 #[cfg(test)]
 mod tests {
-    use crate::model::{Project, ProjectPipeline, ProjectPipelines, test};
+    use crate::model::{test, Project, ProjectPipeline, ProjectPipelines};
 
     #[test]
     fn project_deserialize() {
@@ -196,6 +198,7 @@ mod tests {
             group_id: 1,
             project: test::new_project(),
             pipeline: None,
+            failed_jobs: None,
         };
 
         let json = serde_json::to_string(&value).unwrap();
@@ -210,10 +213,27 @@ mod tests {
             group_id: 1,
             project: test::new_project(),
             pipeline: Some(test::new_pipeline()),
+            failed_jobs: None,
         };
 
         let json = serde_json::to_string(&value).unwrap();
         let expected = "{\"group_id\":1,\"project\":{\"id\":456,\"name\":\"name\",\"web_url\":\"web_url\",\"default_branch\":\"default_branch\",\"topics\":[\"topic\"],\"namespace\":{\"id\":123,\"name\":\"namespace\",\"path\":\"namespace\"}},\"pipeline\":{\"id\":1,\"iid\":2,\"project_id\":3,\"sha\":\"sha\",\"ref\":\"branch\",\"status\":\"running\",\"source\":\"web\",\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"web_url\":\"web_url\"}}";
+
+        assert_eq!(expected, json);
+    }
+
+    #[test]
+    fn project_pipeline_serialize_some_failed_jobs() {
+        let value = ProjectPipeline {
+            group_id: 1,
+            project: test::new_project(),
+            pipeline: None,
+            failed_jobs: Some(vec![test::new_job()]),
+        };
+
+        let json = serde_json::to_string(&value).unwrap();
+        let expected = "{\"group_id\":1,\"project\":{\"id\":456,\"name\":\"name\",\"web_url\":\"web_url\",\"default_branch\":\"default_branch\",\"topics\":[\"topic\"],\"namespace\":{\"id\":123,\"name\":\"namespace\",\"path\":\"namespace\"}},\"failed_jobs\":[{\"id\":1,\"created_at\":\"1970-01-01T00:00:00Z\",\"allow_failure\":false,\"name\":\"name\",\"ref\":\"branch\",\"stage\":\"stage\",\"status\":\"success\",\"web_url\":\"web_url\",\"pipeline\":{\"id\":1,\"iid\":2,\"project_id\":3,\"sha\":\"sha\",\"ref\":\"branch\",\"status\":\"running\",\"source\":\"web\",\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"web_url\":\"web_url\"},\"commit\":{\"id\":\"id\",\"author_name\":\"author_name\",\"committer_name\":\"committer_name\",\"committed_date\":\"1970-01-01T00:00:00Z\",\"title\":\"title\",\"message\":\"message\"},\"user\":{\"id\":123,\"username\":\"username\",\"name\":\"name\",\"state\":\"state\",\"is_admin\":false}}]}";
+
         assert_eq!(expected, json);
     }
 

api/src/model/schedule.rs

@@ -2,7 +2,7 @@ use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 
 use crate::model::user::User;
-use crate::model::{Pipeline, Project};
+use crate::model::{Job, Pipeline, Project};
 use crate::util::deserialize::from_ref;
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -27,6 +27,8 @@ pub struct ScheduleProjectPipeline {
     pub project: Project,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub pipeline: Option<Pipeline>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub failed_jobs: Option<Vec<Job>>,
 }
 
 #[cfg(test)]
@@ -68,6 +70,7 @@ mod tests {
 
         let json = serde_json::to_string(&value).unwrap();
         let expected = "{\"id\":789,\"description\":\"description\",\"ref\":\"branch\",\"cron\":\"cron\",\"cron_timezone\":\"cron_timezone\",\"next_run_at\":\"1970-01-01T00:00:00Z\",\"active\":false,\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"owner\":{\"id\":123,\"username\":\"username\",\"name\":\"name\",\"state\":\"state\",\"is_admin\":false}}";
+
         assert_eq!(expected, json);
     }
 
@@ -78,6 +81,7 @@ mod tests {
             schedule: test::new_schedule(),
             project: test::new_project(),
             pipeline: None,
+            failed_jobs: None,
         };
 
         let json = serde_json::to_string(&value).unwrap();
@@ -93,11 +97,28 @@ mod tests {
             schedule: test::new_schedule(),
             project: test::new_project(),
             pipeline: Some(test::new_pipeline()),
+            failed_jobs: None,
         };
 
         let json = serde_json::to_string(&value).unwrap();
         let expected = "{\"group_id\":1,\"schedule\":{\"id\":789,\"description\":\"description\",\"ref\":\"branch\",\"cron\":\"cron\",\"cron_timezone\":\"cron_timezone\",\"next_run_at\":\"1970-01-01T00:00:00Z\",\"active\":false,\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"owner\":{\"id\":123,\"username\":\"username\",\"name\":\"name\",\"state\":\"state\",\"is_admin\":false}},\"project\":{\"id\":456,\"name\":\"name\",\"web_url\":\"web_url\",\"default_branch\":\"default_branch\",\"topics\":[\"topic\"],\"namespace\":{\"id\":123,\"name\":\"namespace\",\"path\":\"namespace\"}},\"pipeline\":{\"id\":1,\"iid\":2,\"project_id\":3,\"sha\":\"sha\",\"ref\":\"branch\",\"status\":\"running\",\"source\":\"web\",\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"web_url\":\"web_url\"}}";
 
         assert_eq!(expected, json);
     }
+
+    #[test]
+    fn schedule_project_pipeline_serialize_some_failed_jobs() {
+        let value = ScheduleProjectPipeline {
+            group_id: 1,
+            schedule: test::new_schedule(),
+            project: test::new_project(),
+            pipeline: None,
+            failed_jobs: Some(vec![test::new_job()]),
+        };
+
+        let json = serde_json::to_string(&value).unwrap();
+        let expected = "{\"group_id\":1,\"schedule\":{\"id\":789,\"description\":\"description\",\"ref\":\"branch\",\"cron\":\"cron\",\"cron_timezone\":\"cron_timezone\",\"next_run_at\":\"1970-01-01T00:00:00Z\",\"active\":false,\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"owner\":{\"id\":123,\"username\":\"username\",\"name\":\"name\",\"state\":\"state\",\"is_admin\":false}},\"project\":{\"id\":456,\"name\":\"name\",\"web_url\":\"web_url\",\"default_branch\":\"default_branch\",\"topics\":[\"topic\"],\"namespace\":{\"id\":123,\"name\":\"namespace\",\"path\":\"namespace\"}},\"failed_jobs\":[{\"id\":1,\"created_at\":\"1970-01-01T00:00:00Z\",\"allow_failure\":false,\"name\":\"name\",\"ref\":\"branch\",\"stage\":\"stage\",\"status\":\"success\",\"web_url\":\"web_url\",\"pipeline\":{\"id\":1,\"iid\":2,\"project_id\":3,\"sha\":\"sha\",\"ref\":\"branch\",\"status\":\"running\",\"source\":\"web\",\"created_at\":\"1970-01-01T00:00:00Z\",\"updated_at\":\"1970-01-01T00:00:00Z\",\"web_url\":\"web_url\"},\"commit\":{\"id\":\"id\",\"author_name\":\"author_name\",\"committer_name\":\"committer_name\",\"committed_date\":\"1970-01-01T00:00:00Z\",\"title\":\"title\",\"message\":\"message\"},\"user\":{\"id\":123,\"username\":\"username\",\"name\":\"name\",\"state\":\"state\",\"is_admin\":false}}]}";
+
+        assert_eq!(expected, json);
+    }
 }

api/src/project/pipeline_aggregator.rs

@@ -1,19 +1,26 @@
 use crate::error::ApiError;
-use crate::model::{Project, ProjectPipeline, ProjectPipelines};
+use crate::job::JobService;
+use crate::model::{JobStatus, PipelineStatus, Project, ProjectPipeline, ProjectPipelines};
 use crate::pipeline::{sort_by_updated_date, PipelineService};
 use crate::project::ProjectService;
 use crate::util::iter::try_collect_with_buffer;
 
 pub struct PipelineAggregator {
     project_service: ProjectService,
     pipeline_service: PipelineService,
+    job_service: JobService,
 }
 
 impl PipelineAggregator {
-    pub fn new(project_service: ProjectService, pipeline_service: PipelineService) -> Self {
+    pub fn new(
+        project_service: ProjectService,
+        pipeline_service: PipelineService,
+        job_service: JobService,
+    ) -> Self {
         Self {
             project_service,
             pipeline_service,
+            job_service,
         }
     }
 }
@@ -53,10 +60,20 @@ impl PipelineAggregator {
                 None
             };
 
+            let failed_jobs = match pipeline {
+                Some(ref p) if p.status == PipelineStatus::Failed => Some(
+                    self.job_service
+                        .get_jobs(p.project_id, p.id, &[JobStatus::Failed])
+                        .await?,
+                ),
+                _ => None,
+            };
+
             Ok(ProjectPipeline {
                 group_id,
                 project,
                 pipeline,
+                failed_jobs,
             })
         })
         .await
@@ -87,6 +104,7 @@ impl PipelineAggregator {
             } else {
                 Vec::default()
             };
+
             Ok(ProjectPipelines {
                 group_id,
                 project,