Merge pull request #11487 from cmaglie/do-not-delete-unused-packages-index

Do not delete unused 3rd party index files

7 files changed, 37 insertions, 143 deletions

diff --git a/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java b/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java
index 0c949fe1c..c00e91e9d 100644
--- a/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java
+++ b/app/src/cc/arduino/contributions/packages/ui/ContributionManagerUI.java
@@ -140,9 +140,7 @@ public class ContributionManagerUI extends InstallerJDialog {
     installerThread = new Thread(() -> {
       try {
         setProgressVisible(true, "");
-        List<String> downloadedPackageIndexFiles = installer
-            .updateIndex(this::setProgress);
-        installer.deleteUnknownFiles(downloadedPackageIndexFiles);
+        installer.updateIndex(this::setProgress);
         onIndexesUpdated();
         if (contribTable.getCellEditor() != null) {
           contribTable.getCellEditor().stopCellEditing();
diff --git a/app/src/processing/app/Base.java b/app/src/processing/app/Base.java
index 4578038f3..27013566c 100644
--- a/app/src/processing/app/Base.java
+++ b/app/src/processing/app/Base.java
@@ -315,8 +315,7 @@ public class Base {
           BaseNoGui.getPlatform(), gpgDetachedSignatureVerifier);
       ProgressListener progressListener = new ConsoleProgressListener();
 
-      List<String> downloadedPackageIndexFiles = contributionInstaller.updateIndex(progressListener);
-      contributionInstaller.deleteUnknownFiles(downloadedPackageIndexFiles);
+      contributionInstaller.updateIndex(progressListener);
       indexer.parseIndex();
       indexer.syncWithFilesystem();
 
diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java
index 2b6ff4cde..287f18e49 100644
--- a/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java
+++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionInstaller.java
@@ -41,7 +41,6 @@ import org.apache.commons.exec.CommandLine;
 import org.apache.commons.exec.DefaultExecutor;
 import org.apache.commons.exec.Executor;
 import org.apache.commons.exec.PumpStreamHandler;
-import org.apache.commons.io.FilenameUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import processing.app.BaseNoGui;
@@ -284,7 +283,7 @@ public class ContributionInstaller {
     return errors;
   }
 
-  public synchronized List<String> updateIndex(ProgressListener progressListener) {
+  public synchronized void updateIndex(ProgressListener progressListener) {
     MultiStepProgress progress = new MultiStepProgress(1);
 
     final DownloadableContributionsDownloader downloader = new DownloadableContributionsDownloader(BaseNoGui.indexer.getStagingFolder());
@@ -293,14 +292,11 @@ public class ContributionInstaller {
       PreferencesData.getCollection(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS)
     );
     packageIndexURLs.add(Constants.PACKAGE_INDEX_URL);
-    List<String> downloadedPackageIndexFilesAccumulator = new LinkedList<>();
 
     for (String packageIndexURLString : packageIndexURLs) {
       try {
         // Extract the file name from the URL
         final URL packageIndexURL = new URL(packageIndexURLString);
-        String indexFileName = FilenameUtils.getName(packageIndexURL.getPath());
-        downloadedPackageIndexFilesAccumulator.add(BaseNoGui.indexer.getIndexFile(indexFileName).getName());
 
         log.info("Start download and signature check of={}", packageIndexURLs);
         downloader.downloadIndexAndSignature(progress, packageIndexURL, progressListener, signatureVerifier);
@@ -312,22 +308,5 @@ public class ContributionInstaller {
 
     progress.stepDone();
     log.info("Downloaded package index URL={}", packageIndexURLs);
-    return downloadedPackageIndexFilesAccumulator;
-  }
-
-  public synchronized void deleteUnknownFiles(List<String> downloadedPackageIndexFiles) throws IOException {
-    File preferencesFolder = BaseNoGui.indexer.getIndexFile(".").getParentFile();
-    File[] additionalPackageIndexFiles = preferencesFolder.listFiles(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME));
-    if (additionalPackageIndexFiles == null) {
-      return;
-    }
-    log.info("Check unknown files. Additional package index folder files={}, Additional package index url downloaded={}", downloadedPackageIndexFiles, additionalPackageIndexFiles);
-
-    for (File additionalPackageIndexFile : additionalPackageIndexFiles) {
-      if (!downloadedPackageIndexFiles.contains(additionalPackageIndexFile.getName())) {
-        log.info("Delete this unknown file={} because not included in this list={}", additionalPackageIndexFile, additionalPackageIndexFiles);
-        Files.delete(additionalPackageIndexFile.toPath());
-      }
-    }
   }
 }
diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java
index 0a9b19cc3..6b86b0fb2 100644
--- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java
+++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndex.java
@@ -40,7 +40,7 @@ import java.util.stream.Collectors;
 
 public class ContributionsIndex {
 
-  private ArrayList<ContributedPackage> packages = new ArrayList<ContributedPackage>();
+  private ArrayList<ContributedPackage> packages = new ArrayList<>();
   public List<ContributedPackage> getPackages() { return packages; }
 
   public ContributedPackage findPackage(String packageName) {
diff --git a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java
index 060a94e6a..3e68d847e 100644
--- a/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java
+++ b/arduino-core/src/cc/arduino/contributions/packages/ContributionsIndexer.java
@@ -36,6 +36,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import org.apache.commons.compress.utils.IOUtils;
+import org.apache.commons.io.FilenameUtils;
 
 import processing.app.BaseNoGui;
 import processing.app.Platform;
@@ -50,6 +51,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
 import java.util.*;
 import java.util.stream.Collectors;
 
@@ -107,19 +110,14 @@ public class ContributionsIndexer {
     index.getPackages().forEach(pack -> pack.setTrusted(true));
 
     // Overlay 3rd party indexes
-    File[] indexFiles = preferencesFolder.listFiles(new TestPackageIndexFilenameFilter(new PackageIndexFilenameFilter(Constants.DEFAULT_INDEX_FILE_NAME)));
-
-    if (indexFiles != null) {
-      for (File indexFile : indexFiles) {
-        try {
-          mergeContributions(indexFile);
-        } catch (JsonProcessingException e) {
-          System.err.println(format(tr("Skipping contributed index file {0}, parsing error occured:"), indexFile));
-          System.err.println(e);
-        }
+    List<File> indexFiles = get3rdPartyIndexFiles();
+    for (File indexFile : indexFiles) {
+      try {
+        mergeContributions(indexFile);
+      } catch (JsonProcessingException e) {
+        System.err.println(format(tr("Skipping contributed index file {0}, parsing error occured:"), indexFile));
+        System.err.println(e);
       }
-    } else {
-      System.err.println(format(tr("Error reading package indexes folder: {0}\n(maybe a permission problem?)"), preferencesFolder));
     }
 
     // Fill tools and toolsDependency cross references
@@ -146,6 +144,29 @@ public class ContributionsIndexer {
     index.fillCategories();
   }
 
+  private List<File> get3rdPartyIndexFiles() throws MalformedURLException {
+    List<File> indexFiles = new ArrayList<>();
+    for (String urlString : PreferencesData.getCollection(Constants.PREF_BOARDS_MANAGER_ADDITIONAL_URLS)) {
+      final URL url = new URL(urlString);
+      String filename = FilenameUtils.getName(url.getPath());
+      indexFiles.add(getIndexFile(filename));
+    }
+
+    File[] testIndexFiles = preferencesFolder.listFiles((dir, name) -> {
+      if (!new File(dir, name).isFile())
+        return false;
+      if (!name.startsWith("test_package_") || !name.endsWith("_index.json"))
+        return false;
+      return true;
+    });
+    if (testIndexFiles == null) {
+      System.err.println(
+          format(tr("Error reading package indexes folder: {0}\n(maybe a permission problem?)"), preferencesFolder));
+    }
+    indexFiles.addAll(Arrays.asList(testIndexFiles));
+    return indexFiles;
+  }
+
   private void mergeContributions(File indexFile) throws IOException {
     if (!indexFile.exists())
       return;
diff --git a/arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java b/arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java
deleted file mode 100644
index bfc016750..000000000
--- a/arduino-core/src/cc/arduino/contributions/packages/PackageIndexFilenameFilter.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * This file is part of Arduino.
- *
- * Copyright 2015 Arduino LLC (http://www.arduino.cc/)
- *
- * Arduino is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * As a special exception, you may use this file as part of a free software
- * library without restriction.  Specifically, if other files instantiate
- * templates or use macros or inline functions from this file, or you compile
- * this file and link it with other files to produce an executable, this
- * file does not by itself cause the resulting executable to be covered by
- * the GNU General Public License.  This exception does not however
- * invalidate any other reasons why the executable file might be covered by
- * the GNU General Public License.
- */
-
-package cc.arduino.contributions.packages;
-
-import java.io.File;
-import java.io.FilenameFilter;
-
-public class PackageIndexFilenameFilter implements FilenameFilter {
-
-  private final String defaultPackageIndexFileName;
-
-  public PackageIndexFilenameFilter(String defaultPackageIndexFileName) {
-    this.defaultPackageIndexFileName = defaultPackageIndexFileName;
-  }
-
-  @Override
-  public boolean accept(File file, String name) {
-    return new File(file, name).isFile() && !defaultPackageIndexFileName.equals(name) && name.startsWith("package_") && name.endsWith("_index.json");
-  }
-}
diff --git a/arduino-core/src/cc/arduino/contributions/packages/TestPackageIndexFilenameFilter.java b/arduino-core/src/cc/arduino/contributions/packages/TestPackageIndexFilenameFilter.java
deleted file mode 100644
index fdcd08384..000000000
--- a/arduino-core/src/cc/arduino/contributions/packages/TestPackageIndexFilenameFilter.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * This file is part of Arduino.
- *
- * Copyright 2015 Arduino LLC (http://www.arduino.cc/)
- *
- * Arduino is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
- *
- * As a special exception, you may use this file as part of a free software
- * library without restriction.  Specifically, if other files instantiate
- * templates or use macros or inline functions from this file, or you compile
- * this file and link it with other files to produce an executable, this
- * file does not by itself cause the resulting executable to be covered by
- * the GNU General Public License.  This exception does not however
- * invalidate any other reasons why the executable file might be covered by
- * the GNU General Public License.
- */
-
-package cc.arduino.contributions.packages;
-
-import java.io.File;
-import java.io.FilenameFilter;
-
-public class TestPackageIndexFilenameFilter implements FilenameFilter {
-
-  private final FilenameFilter parent;
-
-  public TestPackageIndexFilenameFilter(FilenameFilter parent) {
-    this.parent = parent;
-  }
-
-  public TestPackageIndexFilenameFilter() {
-    this(null);
-  }
-
-  @Override
-  public boolean accept(File file, String name) {
-    boolean result = false;
-    if (parent != null) {
-      result = parent.accept(file, name);
-    }
-    result = result || (new File(file, name).isFile() && name.startsWith("test_package_") && name.endsWith("_index.json"));
-    return result;
-  }
-}