With `EXTSVC_CONFIG_FILE` set, frontend deletes and recreates repos on every restart

Created by: mrnugget

I'm quoting @sqs on Slack here

I ran into problems related to EXTSVC_CONFIG_FILE in dev that might cause pain for customers in prod. Each time the frontend starts up, if EXTSVC_CONFIG_FILE is set, it deletes all external services and recreates them (see handleConfigOverrides)

This means that every repo DB row's sources column becomes invalid, and calls to repoupdater.DefaultClient.RepoExternalServices will return empty for every repo until repo-updater can do a full refresh.

I can work around this in dev, but it seems like this might cause a lot of repo-updater churn in prod. Can someone confirm?

And here is @sqs' proposed diff for a fix:

diff --git a/cmd/frontend/internal/cli/config.go b/cmd/frontend/internal/cli/config.go
index cdbd3e5da..b1083dcf5 100644
--- a/cmd/frontend/internal/cli/config.go
+++ b/cmd/frontend/internal/cli/config.go
@@ -94,39 +94,72 @@ func handleConfigOverrides() error {
                        if err != nil {
                                return errors.Wrap(err, "ExternalServices.List")
                        }
-                       for _, existing := range existing {
-                               err := db.ExternalServices.Delete(ctx, existing.ID)
-                               if err != nil {
-                                       return errors.Wrap(err, "ExternalServices.Delete")
-                               }
-                       }
 
                        extsvc, err := ioutil.ReadFile(overrideExtSvcConfig)
                        if err != nil {
                                return errors.Wrap(err, "reading EXTSVC_CONFIG_FILE")
                        }
-                       var configs map[string][]*json.RawMessage
-                       if err := jsonc.Unmarshal(string(extsvc), &configs); err != nil {
+                       var rawConfigs map[string][]*json.RawMessage
+                       if err := jsonc.Unmarshal(string(extsvc), &rawConfigs); err != nil {
                                return errors.Wrap(err, "parsing EXTSVC_CONFIG_FILE")
                        }
-                       if len(configs) == 0 {
+                       if len(rawConfigs) == 0 {
                                log15.Warn("EXTSVC_CONFIG_FILE contains zero external service configurations")
                        }
-                       for key, cfgs := range configs {
+
+                       // Perform delta update for external services. We don't want to just delete all external
+                       // services and re-add all of them, because that would cause repo-updater to need to
+                       // update repositories and reassociate them with external services each time the
+                       // frontend restarts.
+                       //
+                       // Start out by assuming we will remove all and re-add all.
+                       var (
+                               toAdd    = map[*types.ExternalService]struct{}{}
+                               toRemove = map[*types.ExternalService]struct{}{}
+                               // TODO: support updating in-place instead of remove-then-add (if DisplayName is equal, perhaps?)
+                       )
+                       for _, existing := range existing {
+                               toRemove[existing] = struct{}{}
+                       }
+                       for key, cfgs := range rawConfigs {
                                for i, cfg := range cfgs {
                                        marshaledCfg, err := json.MarshalIndent(cfg, "", "  ")
                                        if err != nil {
                                                return errors.Wrap(err, fmt.Sprintf("marshaling extsvc config ([%v][%v])", key, i))
                                        }
-                                       if err := db.ExternalServices.Create(ctx, confGet, &types.ExternalService{
+                                       toAdd[&types.ExternalService{
                                                Kind:        key,
                                                DisplayName: fmt.Sprintf("%s #%d", key, i+1),
                                                Config:      string(marshaledCfg),
-                                       }); err != nil {
-                                               return errors.Wrap(err, "ExternalServices.Create")
+                                       }] = struct{}{}
+                               }
+                       }
+                       // Now eliminate operations from toAdd/toRemove where the config file and DB describe an
+                       // equivalent external service.
+                       isEquiv := func(a, b *types.ExternalService) bool {
+                               return a.Kind == b.Kind && a.DisplayName == b.DisplayName && a.Config == b.Config
+                       }
+                       for a := range toAdd {
+                               for b := range toRemove {
+                                       if isEquiv(a, b) {
+                                               delete(toAdd, a)
+                                               delete(toRemove, b)
                                        }
                                }
                        }
+
+                       // Apply the delta update.
+                       for extSvc := range toRemove {
+                               err := db.ExternalServices.Delete(ctx, extSvc.ID)
+                               if err != nil {
+                                       return errors.Wrap(err, "ExternalServices.Delete")
+                               }
+                       }
+                       for extSvc := range toAdd {
+                               if err := db.ExternalServices.Create(ctx, confGet, extSvc); err != nil {
+                                       return errors.Wrap(err, "ExternalServices.Create")
+                               }
+                       }
                }
        }
        return nil