feat: check pre-existing feed during feed import

web-app/package.json

@@ -51,7 +51,8 @@
     "recharts": "^2.12.7",
     "redux-persist": "^6.0.0",
     "redux-saga": "^1.2.3",
-    "yup": "^1.3.2"
+    "yup": "^1.3.2",
+    "papaparse": "^5.5.3"
   },
   "peerDependencies": {
     "react": "^17.0.0 || ^18.0.0",

web-app/public/locales/en/feeds.json

@@ -46,6 +46,7 @@
     "errorSubmitting": "An error occurred while submitting the form.",
     "submittingFeed": "Submitting the feed...",
     "errorUrl": "The URL must start with a valid protocol: http:// or https://",
+    "feedAlreadyExists": "This feed URL already exists in the Mobility Database: ",
     "unofficialDesc": "Why was this feed created?",
     "unofficialDescPlaceholder": "Does this feed exist for research purposes, a specific app, etc?",
     "updateFreq": "How often is this feed updated?",

web-app/src/app/screens/FeedSubmission/Form/FirstStep.tsx

@@ -21,7 +21,10 @@ import {
 import { type YesNoFormInput, type FeedSubmissionFormFormInput } from '.';
 import { useEffect } from 'react';
 import { useTranslation } from 'react-i18next';
-import { isValidFeedLink } from '../../../services/feeds/utils';
+import {
+  isValidFeedLink,
+  checkFeedUrlExistsInCsv,
+} from '../../../services/feeds/utils';
 import FormLabelDescription from './components/FormLabelDescription';
 
 export interface FeedSubmissionFormFormInputFirstStep {
@@ -42,6 +45,8 @@ interface FormFirstStepProps {
   setNumberOfSteps: (numberOfSteps: YesNoFormInput) => void;
 }
 
+const scheduleFeedURLPrefix = 'https://mobilitydatabase.org/feeds/gtfs/';
+
 export default function FormFirstStep({
   initialValues,
   submitFormData,
@@ -267,18 +272,50 @@ export default function FormFirstStep({
                 <Controller
                   rules={{
                     required: t('form.feedLinkRequired'),
-                    validate: (value) =>
-                      isValidFeedLink(value ?? '') || t('form.errorUrl'),
+                    validate: async (value) => {
+                      if (!isValidFeedLink(value ?? '')) {
+                        return t('form.errorUrl');
+                      }
+                      const exists = await checkFeedUrlExistsInCsv(value ?? '');
+                      if (typeof exists === 'string' && exists.length > 0) {
+                        return `Feed Exists:${exists}`;
+                      }
+                      return true;
+                    },
                   }}
                   control={control}
                   name='feedLink'
                   render={({ field }) => (
                     <TextField
                       data-cy='feedLink'
                       className='md-small-input'
-                      helperText={errors.feedLink?.message ?? ''}
                       error={errors.feedLink !== undefined}
                       {...field}
+                      helperText={
+                        typeof errors.feedLink?.message === 'string' &&
+                        errors.feedLink?.message?.startsWith('Feed Exists:') ? (
+                          <span>
+                            {t('form.feedAlreadyExists')}
+                            <a
+                              href={errors.feedLink.message.replace(
+                                'Feed Exists:',
+                                `${scheduleFeedURLPrefix}`,
+                              )}
+                              target='_blank'
+                              rel='noopener noreferrer'
+                            >
+                              {t(
+                                errors.feedLink.message.replace(
+                                  'Feed Exists:',
+                                  '',
+                                ),
+                              )}
+                            </a>
+                          </span>
+                        ) : (
+                          errors.feedLink?.message ?? ''
+                        )
+                      }
                     />
                   )}
                 />

web-app/src/app/screens/FeedSubmission/Form/SecondStepRealtime.tsx

@@ -10,7 +10,10 @@ import { type SubmitHandler, Controller, useForm } from 'react-hook-form';
 import { type AuthTypes, type FeedSubmissionFormFormInput } from '.';
 import { useEffect } from 'react';
 import { useTranslation } from 'react-i18next';
-import { isValidFeedLink } from '../../../services/feeds/utils';
+import {
+  isValidFeedLink,
+  checkFeedUrlExistsInCsv,
+} from '../../../services/feeds/utils';
 
 export interface FeedSubmissionFormInputSecondStepRT {
   tripUpdates: string;
@@ -32,6 +35,8 @@ interface FormSecondStepRTProps {
   handleBack: (formData: Partial<FeedSubmissionFormFormInput>) => void;
 }
 
+const realtimeFeedURLPrefix = 'https://mobilitydatabase.org/feeds/gtfs_rt/';
+
 export default function FormSecondStepRT({
   initialValues,
   submitFormData,
@@ -125,14 +130,52 @@ export default function FormSecondStepRT({
               <Controller
                 control={control}
                 name='serviceAlerts'
-                rules={{ validate: () => gtfsRtLinkValidation('sa') }}
+                rules={{
+                  validate: async (value) => {
+                    const atLeastOneFeed = gtfsRtLinkValidation('sa');
+                    if (atLeastOneFeed !== true) {
+                      return atLeastOneFeed;
+                    }
+                    const exists = await checkFeedUrlExistsInCsv(value ?? '');
+                    if (typeof exists === 'string' && exists.length > 0) {
+                      return `Feed Exists:${exists}`;
+                    }
+                    return true;
+                  },
+                }}
                 render={({ field }) => (
                   <TextField
                     className='md-small-input'
                     {...field}
-                    helperText={errors.serviceAlerts?.message ?? ''}
                     error={errors.serviceAlerts !== undefined}
                     data-cy='serviceAlertFeed'
+                    helperText={
+                      typeof errors.serviceAlerts?.message === 'string' &&
+                      errors.serviceAlerts?.message?.startsWith(
+                        'Feed Exists:',
+                      ) ? (
+                        <span>
+                          {t('form.feedAlreadyExists')}
+                          <a
+                            href={errors.serviceAlerts.message.replace(
+                              'Feed Exists:',
+                              `${realtimeFeedURLPrefix}`,
+                            )}
+                            target='_blank'
+                            rel='noopener noreferrer'
+                          >
+                            {t(
+                              errors.serviceAlerts.message.replace(
+                                'Feed Exists:',
+                                '',
+                              ),
+                            )}
+                          </a>
+                        </span>
+                      ) : (
+                        errors.serviceAlerts?.message ?? ''
+                      )
+                    }
                   />
                 )}
               />
@@ -181,13 +224,51 @@ export default function FormSecondStepRT({
               <Controller
                 control={control}
                 name='tripUpdates'
-                rules={{ validate: () => gtfsRtLinkValidation('tu') }}
+                rules={{
+                  validate: async (value) => {
+                    const atLeastOneFeed = gtfsRtLinkValidation('tu');
+                    if (atLeastOneFeed !== true) {
+                      return atLeastOneFeed;
+                    }
+                    const exists = await checkFeedUrlExistsInCsv(value ?? '');
+                    if (typeof exists === 'string' && exists.length > 0) {
+                      return `Feed Exists:${exists}`;
+                    }
+                    return true;
+                  },
+                }}
                 render={({ field }) => (
                   <TextField
                     className='md-small-input'
                     {...field}
-                    helperText={errors.tripUpdates?.message ?? ''}
                     error={errors.tripUpdates !== undefined}
+                    helperText={
+                      typeof errors.tripUpdates?.message === 'string' &&
+                      errors.tripUpdates?.message?.startsWith(
+                        'Feed Exists:',
+                      ) ? (
+                        <span>
+                          {t('form.feedAlreadyExists')}
+                          <a
+                            href={errors.tripUpdates.message.replace(
+                              'Feed Exists:',
+                              `${realtimeFeedURLPrefix}`,
+                            )}
+                            target='_blank'
+                            rel='noopener noreferrer'
+                          >
+                            {t(
+                              errors.tripUpdates.message.replace(
+                                'Feed Exists:',
+                                '',
+                              ),
+                            )}
+                          </a>
+                        </span>
+                      ) : (
+                        errors.tripUpdates?.message ?? ''
+                      )
+                    }
                   />
                 )}
               />
@@ -236,13 +317,51 @@ export default function FormSecondStepRT({
               <Controller
                 control={control}
                 name='vehiclePositions'
-                rules={{ validate: () => gtfsRtLinkValidation('vp') }}
+                rules={{
+                  validate: async (value) => {
+                    const atLeastOneFeed = gtfsRtLinkValidation('vp');
+                    if (atLeastOneFeed !== true) {
+                      return atLeastOneFeed;
+                    }
+                    const exists = await checkFeedUrlExistsInCsv(value ?? '');
+                    if (typeof exists === 'string' && exists.length > 0) {
+                      return `Feed Exists:${exists}`;
+                    }
+                    return true;
+                  },
+                }}
                 render={({ field }) => (
                   <TextField
                     className='md-small-input'
                     {...field}
-                    helperText={errors.vehiclePositions?.message ?? ''}
                     error={errors.vehiclePositions !== undefined}
+                    helperText={
+                      typeof errors.vehiclePositions?.message === 'string' &&
+                      errors.vehiclePositions?.message?.startsWith(
+                        'Feed Exists:',
+                      ) ? (
+                        <span>
+                          {t('form.feedAlreadyExists')}
+                          <a
+                            href={errors.vehiclePositions.message.replace(
+                              'Feed Exists:',
+                              `${realtimeFeedURLPrefix}`,
+                            )}
+                            target='_blank'
+                            rel='noopener noreferrer'
+                          >
+                            {t(
+                              errors.vehiclePositions.message.replace(
+                                'Feed Exists:',
+                                '',
+                              ),
+                            )}
+                          </a>
+                        </span>
+                      ) : (
+                        errors.vehiclePositions?.message ?? ''
+                      )
+                    }
                   />
                 )}
               />

web-app/src/app/services/feeds/utils.ts

@@ -1,3 +1,4 @@
+import Papa from 'papaparse';
 import { getEmojiFlag, type TCountryCode, languages } from 'countries-list';
 import { type paths, type components } from './types';
 
@@ -150,3 +151,35 @@ export const langCodeToName = (code: string): string => {
   const lang = languages[primary as keyof typeof languages];
   return lang?.name ?? code.toUpperCase();
 };
+
+/**
+ * Checks if a feed URL exists in the urls.direct_download column of the feeds_v2.csv file.
+ * @param feedUrl The URL to check for existence.
+ * @param csvUrl The CSV file URL (default: feeds_v2.csv from Mobility Database)
+ * @returns Promise<boolean> true if exists, false otherwise
+ */
+export async function checkFeedUrlExistsInCsv(
+  feedUrl: string,
+  csvUrl = 'https://files.mobilitydatabase.org/feeds_v2.csv',
+): Promise<string | null> {
+  try {
+    const response = await fetch(csvUrl);
+    if (!response.ok) throw new Error('Failed to fetch CSV');
+    const csvText = await response.text();
+    const parsed = Papa.parse<FeedCsvRow>(csvText, { header: true });
+    if (parsed.data == null || !Array.isArray(parsed.data)) return null;
+    const rows = parsed.data;
+    const match = rows.find((row) => row['urls.direct_download'] === feedUrl);
+    return typeof match?.id === 'string' ? match.id : null;
+  } catch (e) {
+    return null;
+  }
+}
+
+/**
+ * FeedCsvRow interface with fields
+ */
+interface FeedCsvRow {
+  'urls.direct_download'?: string;
+  id: string;
+}

web-app/yarn.lock

@@ -11600,6 +11600,11 @@ pac-resolver@^7.0.0:
     ip "^1.1.8"
     netmask "^2.0.2"
 
+papaparse@^5.5.3:
+  version "5.5.3"
+  resolved "https://registry.yarnpkg.com/papaparse/-/papaparse-5.5.3.tgz#07f8994dec516c6dab266e952bed68e1de59fa9a"
+  integrity sha512-5QvjGxYVjxO59MGU2lHVYpRWBBtKHnlIAcSe1uNFCkkptUh63NFRj0FJQm7nR67puEruUci/ZkjmEFrjCAyP4A==
+
 param-case@^3.0.4:
   version "3.0.4"
   resolved "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz"