From da69b470694011b925093c725ed72cc94c1acfe1 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 17:19:29 +0200 Subject: [PATCH 01/15] Replaced copyRecursive with go-paths library calls --- .../builder/test/helper_tools_downloader.go | 84 +++---------------- 1 file changed, 10 insertions(+), 74 deletions(-) diff --git a/legacy/builder/test/helper_tools_downloader.go b/legacy/builder/test/helper_tools_downloader.go index 133122c4ab6..d15223e3f4b 100644 --- a/legacy/builder/test/helper_tools_downloader.go +++ b/legacy/builder/test/helper_tools_downloader.go @@ -22,7 +22,6 @@ import ( "net/http" "os" "os/exec" - "path/filepath" "runtime" "strings" "testing" @@ -465,8 +464,7 @@ func downloadAndUnpackCore(core Core, url string, targetPath *paths.Path) error if err := packagerPath.MkdirAll(); err != nil { return errors.WithStack(err) } - err = copyRecursive(unpackFolder.Join(files[0].Name()), targetPath.Join(core.Maintainer, core.Arch)) - if err != nil { + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(targetPath.Join(core.Maintainer, core.Arch)); err != nil { return errors.WithStack(err) } } else { @@ -474,8 +472,7 @@ func downloadAndUnpackCore(core Core, url string, targetPath *paths.Path) error return errors.WithStack(err) } for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), targetPath.Join(core.Maintainer, core.Arch, file.Name())) - if err != nil { + if err := unpackFolder.Join(file.Name()).CopyDirTo(targetPath.Join(core.Maintainer, core.Arch, file.Name())); err != nil { return errors.WithStack(err) } } @@ -510,8 +507,7 @@ func downloadAndUnpackBoardManagerCore(core Core, url string, targetPath *paths. if err := corePath.MkdirAll(); err != nil { return errors.WithStack(err) } - err = copyRecursive(unpackFolder.Join(files[0].Name()), corePath.Join(core.Version)) - if err != nil { + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(corePath.Join(core.Version)); err != nil { return errors.WithStack(err) } } else { @@ -519,8 +515,7 @@ func downloadAndUnpackBoardManagerCore(core Core, url string, targetPath *paths. return errors.WithStack(err) } for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), corePath.Join(core.Version, file.Name())) - if err != nil { + if err := unpackFolder.Join(file.Name()).CopyDirTo(corePath.Join(core.Version, file.Name())); err != nil { return errors.WithStack(err) } } @@ -548,8 +543,7 @@ func downloadAndUnpackBoardsManagerTool(tool Tool, url string, targetPath *paths if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name).MkdirAll(); err != nil { return errors.WithStack(err) } - err = copyRecursive(unpackFolder.Join(files[0].Name()), targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)) - if err != nil { + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)); err != nil { return errors.WithStack(err) } } else { @@ -557,8 +551,7 @@ func downloadAndUnpackBoardsManagerTool(tool Tool, url string, targetPath *paths return errors.WithStack(err) } for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version, file.Name())) - if err != nil { + if err := unpackFolder.Join(file.Name()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version, file.Name())); err != nil { return errors.WithStack(err) } } @@ -595,17 +588,16 @@ func downloadAndUnpackTool(tool Tool, url string, targetPath *paths.Path, delete if err := toolPath.MkdirAll(); err != nil { return errors.WithStack(err) } - err = copyRecursive(unpackFolder.Join(files[0].Name()), toolPath.Join(tool.Version)) - if err != nil { + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(toolPath.Join(tool.Version)); err != nil { return errors.WithStack(err) } } else { + unpackFolder.CopyDirTo(toolPath.Join(tool.Version)) if err := toolPath.Join(tool.Version).MkdirAll(); err != nil { return errors.WithStack(err) } for _, file := range files { - err = copyRecursive(unpackFolder.Join(file.Name()), toolPath.Join(tool.Version, file.Name())) - if err != nil { + if err := unpackFolder.Join(file.Name()).CopyTo(toolPath.Join(tool.Version, file.Name())); err != nil { return errors.WithStack(err) } } @@ -773,65 +765,9 @@ func downloadAndUnpackLibrary(library Library, url string, targetPath *paths.Pat } } - err = copyRecursive(unpackFolder.Join(files[0].Name()), libPath) - if err != nil { + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(libPath); err != nil { return errors.WithStack(err) } return nil } - -func copyRecursive(from, to *paths.Path) error { - copyFunc := func(currentPath string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - rel, err := filepath.Rel(from.String(), currentPath) - if err != nil { - return errors.WithStack(err) - } - targetPath := filepath.Join(to.String(), rel) - if info.IsDir() { - err := os.MkdirAll(targetPath, info.Mode()) - if err != nil { - return errors.WithStack(err) - } - } else if info.Mode().IsRegular() { - fromFile, err := os.Open(currentPath) - if err != nil { - return errors.WithStack(err) - } - defer fromFile.Close() - targetFile, err := os.Create(targetPath) - if err != nil { - return errors.WithStack(err) - } - defer targetFile.Close() - _, err = io.Copy(targetFile, fromFile) - if err != nil { - return errors.WithStack(err) - } - err = os.Chmod(targetPath, info.Mode()) - if err != nil { - return errors.WithStack(err) - } - } else if info.Mode()&os.ModeSymlink == os.ModeSymlink { - linkedFile, err := os.Readlink(currentPath) - if err != nil { - return errors.WithStack(err) - } - fromFile := filepath.Join(filepath.Dir(targetPath), linkedFile) - err = os.Symlink(fromFile, targetPath) - if err != nil { - return errors.WithStack(err) - } - } else { - return errors.Errorf("unable to copy file " + currentPath) - } - - return nil - } - err := gohasissues.Walk(from.String(), copyFunc) - return errors.WithStack(err) -} From 30a0486e0459254b4173e4fe5d28104bb2bd9873 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 17:21:06 +0200 Subject: [PATCH 02/15] Simplified download functions --- .../builder/test/helper_tools_downloader.go | 88 +++++-------------- 1 file changed, 21 insertions(+), 67 deletions(-) diff --git a/legacy/builder/test/helper_tools_downloader.go b/legacy/builder/test/helper_tools_downloader.go index d15223e3f4b..56ea214a0fe 100644 --- a/legacy/builder/test/helper_tools_downloader.go +++ b/legacy/builder/test/helper_tools_downloader.go @@ -454,30 +454,17 @@ func downloadAndUnpackCore(core Core, url string, targetPath *paths.Path) error packagerPath := targetPath.Join(core.Maintainer) corePath := targetPath.Join(core.Maintainer, core.Arch) + if err := packagerPath.MkdirAll(); err != nil { + return errors.WithStack(err) + } if corePath.Exist() { if err := corePath.RemoveAll(); err != nil { return errors.WithStack(err) } } - - if len(files) == 1 && files[0].IsDir() { - if err := packagerPath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(targetPath.Join(core.Maintainer, core.Arch)); err != nil { - return errors.WithStack(err) - } - } else { - if err := targetPath.Join(core.Maintainer, core.Arch).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - if err := unpackFolder.Join(file.Name()).CopyDirTo(targetPath.Join(core.Maintainer, core.Arch, file.Name())); err != nil { - return errors.WithStack(err) - } - } + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(corePath); err != nil { + return errors.WithStack(err) } - return nil } @@ -502,25 +489,12 @@ func downloadAndUnpackBoardManagerCore(core Core, url string, targetPath *paths. return errors.WithStack(err) } } - - if len(files) == 1 && files[0].IsDir() { - if err := corePath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(corePath.Join(core.Version)); err != nil { - return errors.WithStack(err) - } - } else { - if err := corePath.Join(core.Version).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - if err := unpackFolder.Join(file.Name()).CopyDirTo(corePath.Join(core.Version, file.Name())); err != nil { - return errors.WithStack(err) - } - } + if err := corePath.MkdirAll(); err != nil { + return errors.WithStack(err) + } + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(corePath.Join(core.Version)); err != nil { + return errors.WithStack(err) } - return nil } @@ -539,24 +513,12 @@ func downloadAndUnpackBoardsManagerTool(tool Tool, url string, targetPath *paths } defer unpackFolder.RemoveAll() - if len(files) == 1 && files[0].IsDir() { - if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name).MkdirAll(); err != nil { - return errors.WithStack(err) - } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)); err != nil { - return errors.WithStack(err) - } - } else { - if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - if err := unpackFolder.Join(file.Name()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version, file.Name())); err != nil { - return errors.WithStack(err) - } - } + if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name).MkdirAll(); err != nil { + return errors.WithStack(err) + } + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)); err != nil { + return errors.WithStack(err) } - return nil } @@ -583,24 +545,16 @@ func downloadAndUnpackTool(tool Tool, url string, targetPath *paths.Path, delete } } } - + if err := toolPath.MkdirAll(); err != nil { + return errors.WithStack(err) + } + destDir := toolPath.Join(tool.Version) if len(files) == 1 && files[0].IsDir() { - if err := toolPath.MkdirAll(); err != nil { - return errors.WithStack(err) - } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(toolPath.Join(tool.Version)); err != nil { + if err := unpackFolder.Join(files[0].Name()).CopyDirTo(destDir); err != nil { return errors.WithStack(err) } } else { - unpackFolder.CopyDirTo(toolPath.Join(tool.Version)) - if err := toolPath.Join(tool.Version).MkdirAll(); err != nil { - return errors.WithStack(err) - } - for _, file := range files { - if err := unpackFolder.Join(file.Name()).CopyTo(toolPath.Join(tool.Version, file.Name())); err != nil { - return errors.WithStack(err) - } - } + unpackFolder.CopyDirTo(destDir) } return nil From 54864ed2e11c18c77bc8fbe7d56137f9eb2810eb Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 17:34:25 +0200 Subject: [PATCH 03/15] Removed no more needed functions --- legacy/builder/gohasissues/go_has_issues.go | 62 --------------------- 1 file changed, 62 deletions(-) diff --git a/legacy/builder/gohasissues/go_has_issues.go b/legacy/builder/gohasissues/go_has_issues.go index 316308de429..597fbd2107c 100644 --- a/legacy/builder/gohasissues/go_has_issues.go +++ b/legacy/builder/gohasissues/go_has_issues.go @@ -19,70 +19,8 @@ import ( "io/fs" "os" "path/filepath" - "sort" ) -func Walk(root string, walkFn filepath.WalkFunc) error { - info, err := os.Stat(root) - if err != nil { - return walkFn(root, nil, err) - } - return walk(root, info, walkFn) -} - -func walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error { - err := walkFn(path, info, nil) - if err != nil { - if info.IsDir() && err == filepath.SkipDir { - return nil - } - return err - } - - if !info.IsDir() { - return nil - } - - names, err := readDirNames(path) - if err != nil { - return walkFn(path, info, err) - } - - for _, name := range names { - filename := filepath.Join(path, name) - fileInfo, err := os.Stat(filename) - if err != nil { - if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { - return err - } - } else { - err = walk(filename, fileInfo, walkFn) - if err != nil { - if !fileInfo.IsDir() || err != filepath.SkipDir { - return err - } - } - } - } - return nil -} - -// readDirNames reads the directory named by dirname and returns -// a sorted list of directory entries. -func readDirNames(dirname string) ([]string, error) { - f, err := os.Open(dirname) - if err != nil { - return nil, err - } - names, err := f.Readdirnames(-1) - f.Close() - if err != nil { - return nil, err - } - sort.Strings(names) - return names, nil -} - func ReadDir(dirname string) ([]os.FileInfo, error) { entries, err := os.ReadDir(dirname) if err != nil { From a777628837e2c11b29da6142d58db14154ae359e Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 17:39:29 +0200 Subject: [PATCH 04/15] Replaced some calls with go-paths library --- legacy/builder/test/helper_tools_downloader.go | 15 +++++++-------- ...ut_build_path_if_build_options_changed_test.go | 5 ++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/legacy/builder/test/helper_tools_downloader.go b/legacy/builder/test/helper_tools_downloader.go index 56ea214a0fe..c32042c0c15 100644 --- a/legacy/builder/test/helper_tools_downloader.go +++ b/legacy/builder/test/helper_tools_downloader.go @@ -27,7 +27,6 @@ import ( "testing" "github.com/arduino/arduino-cli/legacy/builder/constants" - "github.com/arduino/arduino-cli/legacy/builder/gohasissues" "github.com/arduino/go-paths-helper" "github.com/arduino/go-properties-orderedmap" "github.com/pkg/errors" @@ -462,7 +461,7 @@ func downloadAndUnpackCore(core Core, url string, targetPath *paths.Path) error return errors.WithStack(err) } } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(corePath); err != nil { + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(corePath); err != nil { return errors.WithStack(err) } return nil @@ -492,7 +491,7 @@ func downloadAndUnpackBoardManagerCore(core Core, url string, targetPath *paths. if err := corePath.MkdirAll(); err != nil { return errors.WithStack(err) } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(corePath.Join(core.Version)); err != nil { + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(corePath.Join(core.Version)); err != nil { return errors.WithStack(err) } return nil @@ -516,7 +515,7 @@ func downloadAndUnpackBoardsManagerTool(tool Tool, url string, targetPath *paths if err := targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name).MkdirAll(); err != nil { return errors.WithStack(err) } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)); err != nil { + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(targetPath.Join(tool.Package, constants.FOLDER_TOOLS, tool.Name, tool.Version)); err != nil { return errors.WithStack(err) } return nil @@ -550,7 +549,7 @@ func downloadAndUnpackTool(tool Tool, url string, targetPath *paths.Path, delete } destDir := toolPath.Join(tool.Version) if len(files) == 1 && files[0].IsDir() { - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(destDir); err != nil { + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(destDir); err != nil { return errors.WithStack(err) } } else { @@ -560,7 +559,7 @@ func downloadAndUnpackTool(tool Tool, url string, targetPath *paths.Path, delete return nil } -func downloadAndUnpack(url string) (*paths.Path, []os.FileInfo, error) { +func downloadAndUnpack(url string) (*paths.Path, paths.PathList, error) { fmt.Fprintln(os.Stderr, "Downloading "+url) unpackFolder, err := paths.MkTempDir("", "arduino-builder-tool") @@ -597,7 +596,7 @@ func downloadAndUnpack(url string) (*paths.Path, []os.FileInfo, error) { archiveFilePath.Remove() - files, err := gohasissues.ReadDir(unpackFolder.String()) + files, err := unpackFolder.ReadDir() if err != nil { return nil, nil, errors.WithStack(err) } @@ -719,7 +718,7 @@ func downloadAndUnpackLibrary(library Library, url string, targetPath *paths.Pat } } - if err := unpackFolder.Join(files[0].Name()).CopyDirTo(libPath); err != nil { + if err := unpackFolder.Join(files[0].Base()).CopyDirTo(libPath); err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go b/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go index 2e28b3eff07..e46a66a7dbb 100644 --- a/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go +++ b/legacy/builder/test/wipeout_build_path_if_build_options_changed_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/arduino/arduino-cli/legacy/builder" - "github.com/arduino/arduino-cli/legacy/builder/gohasissues" "github.com/arduino/arduino-cli/legacy/builder/types" "github.com/stretchr/testify/require" ) @@ -48,7 +47,7 @@ func TestWipeoutBuildPathIfBuildOptionsChanged(t *testing.T) { NoError(t, err) require.True(t, exist) - files, err := gohasissues.ReadDir(buildPath.String()) + files, err := buildPath.ReadDir() NoError(t, err) require.Equal(t, 0, len(files)) @@ -80,7 +79,7 @@ func TestWipeoutBuildPathIfBuildOptionsChangedNoPreviousBuildOptions(t *testing. NoError(t, err) require.True(t, exist) - files, err := gohasissues.ReadDir(buildPath.String()) + files, err := buildPath.ReadDir() NoError(t, err) require.Equal(t, 1, len(files)) From 5b6fde242aaabe1a5325cc693e03cea8798174f8 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 17:53:07 +0200 Subject: [PATCH 05/15] Replaced some calls with go-paths library --- legacy/builder/builder_utils/utils.go | 29 ++++++----- legacy/builder/gohasissues/go_has_issues.go | 57 --------------------- legacy/builder/utils/utils.go | 43 ---------------- 3 files changed, 15 insertions(+), 114 deletions(-) delete mode 100644 legacy/builder/gohasissues/go_has_issues.go diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index e7b0d56fe41..d1ab6b3bbce 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -39,30 +39,31 @@ import ( var tr = i18n.Tr func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { - files, err := utils.ReadDirFiltered(sourcePath, utils.FilterFiles()) + allFiles, err := paths.New(sourcePath).ReadDir() if err != nil { return nil, errors.WithStack(err) } + files := allFiles.Clone() + files.FilterOutDirs() var sources []string for _, file := range files { - sources = append(sources, filepath.Join(sourcePath, file.Name())) + sources = append(sources, filepath.Join(sourcePath, file.Base())) } if recurse { - folders, err := utils.ReadDirFiltered(sourcePath, utils.FilterDirs) - if err != nil { - return nil, errors.WithStack(err) - } - + folders := allFiles.Clone() + folders.FilterOutHiddenFiles() + folders.FilterDirs() for _, folder := range folders { - if !utils.IsSCCSOrHiddenFile(folder) { - // Skip SCCS directories as they do not influence the build and can be very large - otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Name()), recurse) - if err != nil { - return nil, errors.WithStack(err) - } - sources = append(sources, otherSources...) + // Skip SCCS directories as they do not influence the build and can be very large + if utils.SOURCE_CONTROL_FOLDERS[folder.Base()] { + continue + } + otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Base()), recurse) + if err != nil { + return nil, errors.WithStack(err) } + sources = append(sources, otherSources...) } } diff --git a/legacy/builder/gohasissues/go_has_issues.go b/legacy/builder/gohasissues/go_has_issues.go deleted file mode 100644 index 597fbd2107c..00000000000 --- a/legacy/builder/gohasissues/go_has_issues.go +++ /dev/null @@ -1,57 +0,0 @@ -// This file is part of arduino-cli. -// -// Copyright 2020 ARDUINO SA (http://www.arduino.cc/) -// -// This software is released under the GNU General Public License version 3, -// which covers the main part of arduino-cli. -// The terms of this license can be found at: -// https://www.gnu.org/licenses/gpl-3.0.en.html -// -// You can be released from the requirements of the above licenses by purchasing -// a commercial license. Buying such a license is mandatory if you want to -// modify or otherwise use the software for commercial activities involving the -// Arduino software without disclosing the source code of your own applications. -// To purchase a commercial license, send an email to license@arduino.cc. - -package gohasissues - -import ( - "io/fs" - "os" - "path/filepath" -) - -func ReadDir(dirname string) ([]os.FileInfo, error) { - entries, err := os.ReadDir(dirname) - if err != nil { - return nil, err - } - - infos := make([]fs.FileInfo, 0, len(entries)) - for _, entry := range entries { - info, err := entry.Info() - if err != nil { - return nil, err - } - - info, err = resolveSymlink(dirname, info) - if err != nil { - // unresolvable symlinks should be skipped silently - continue - } - infos = append(infos, info) - } - - return infos, nil -} - -func resolveSymlink(parentFolder string, info os.FileInfo) (os.FileInfo, error) { - if !isSymlink(info) { - return info, nil - } - return os.Stat(filepath.Join(parentFolder, info.Name())) -} - -func isSymlink(info os.FileInfo) bool { - return info.Mode()&os.ModeSymlink != 0 -} diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 3efc8b73d6d..b05c409c488 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -29,7 +29,6 @@ import ( "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" - "github.com/arduino/arduino-cli/legacy/builder/gohasissues" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" @@ -43,48 +42,6 @@ type filterFiles func([]os.FileInfo) []os.FileInfo var tr = i18n.Tr -func ReadDirFiltered(folder string, fn filterFiles) ([]os.FileInfo, error) { - files, err := gohasissues.ReadDir(folder) - if err != nil { - return nil, errors.WithStack(err) - } - return fn(files), nil -} - -func FilterDirs(files []os.FileInfo) []os.FileInfo { - var filtered []os.FileInfo - for _, info := range files { - if info.IsDir() { - filtered = append(filtered, info) - } - } - return filtered -} - -func FilterFilesWithExtensions(extensions ...string) filterFiles { - return func(files []os.FileInfo) []os.FileInfo { - var filtered []os.FileInfo - for _, file := range files { - if !file.IsDir() && slices.Contains(extensions, filepath.Ext(file.Name())) { - filtered = append(filtered, file) - } - } - return filtered - } -} - -func FilterFiles() filterFiles { - return func(files []os.FileInfo) []os.FileInfo { - var filtered []os.FileInfo - for _, file := range files { - if !file.IsDir() { - filtered = append(filtered, file) - } - } - return filtered - } -} - var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} // FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name From 6bf0ec5829d3765ae9ebfd338d44c797f291456d Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 18:01:13 +0200 Subject: [PATCH 06/15] Isolated some function into the (soon-to-be-removed) CMake generator This change will not break the CMake generator but will remove the CopyDir implementation from the public API space. --- legacy/builder/create_cmake_rule.go | 151 ++++++++++++++++++++++++++-- legacy/builder/utils/utils.go | 132 ------------------------ 2 files changed, 143 insertions(+), 140 deletions(-) diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 9cc38434286..9a2bbecef60 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -17,6 +17,8 @@ package builder import ( "fmt" + "io" + "os" "path/filepath" "regexp" "strings" @@ -39,6 +41,130 @@ type ExportProjectCMake struct { var lineMatcher = regexp.MustCompile(`^#line\s\d+\s"`) func (s *ExportProjectCMake) Run(ctx *types.Context) error { + // copies the contents of the file named src to the file named + // by dst. The file will be created if it does not already exist. If the + // destination file exists, all it's contents will be replaced by the contents + // of the source file. The file mode will be copied from the source and + // the copied data is synced/flushed to stable storage. + // TODO: Replace with call to go-paths-helper... + copyFile := func(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + if e := out.Close(); e != nil { + err = e + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return + } + + err = out.Sync() + if err != nil { + return + } + + si, err := os.Stat(src) + if err != nil { + return + } + err = os.Chmod(dst, si.Mode()) + if err != nil { + return + } + + return + } + + // recursively copies a directory tree, attempting to preserve permissions. + // Source directory must exist, destination directory must *not* exist. + // Symlinks are ignored and skipped. + // TODO: Replace with call to go-paths-helper... + var copyDir func(src string, dst string, extensions []string) (err error) + copyDir = func(src string, dst string, extensions []string) (err error) { + isAcceptedExtension := func(ext string) bool { + ext = strings.ToLower(ext) + for _, valid := range extensions { + if ext == valid { + return true + } + } + return false + } + + src = filepath.Clean(src) + dst = filepath.Clean(dst) + + si, err := os.Stat(src) + if err != nil { + return err + } + if !si.IsDir() { + return fmt.Errorf(tr("source is not a directory")) + } + + _, err = os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + return + } + if err == nil { + return fmt.Errorf(tr("destination already exists")) + } + + err = os.MkdirAll(dst, si.Mode()) + if err != nil { + return + } + + entries, err := os.ReadDir(src) + if err != nil { + return + } + + for _, dirEntry := range entries { + entry, scopeErr := dirEntry.Info() + if scopeErr != nil { + return + } + + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + + if entry.IsDir() { + err = copyDir(srcPath, dstPath, extensions) + if err != nil { + return + } + } else { + // Skip symlinks. + if entry.Mode()&os.ModeSymlink != 0 { + continue + } + + if !isAcceptedExtension(filepath.Ext(srcPath)) { + continue + } + + err = copyFile(srcPath, dstPath) + if err != nil { + return + } + } + } + + return + } + var validExportExtensions = []string{".a", ".properties"} for ext := range globals.SourceFilesValidExtensions { validExportExtensions = append(validExportExtensions, ext) @@ -76,7 +202,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { // Copy used libraries in the correct folder libDir := libBaseFolder.Join(library.DirName) mcu := ctx.BuildProperties.Get(constants.BUILD_PROPERTIES_BUILD_MCU) - utils.CopyDir(library.InstallDir.String(), libDir.String(), validExportExtensions) + copyDir(library.InstallDir.String(), libDir.String(), validExportExtensions) // Read cmake options if available isStaticLib := true @@ -106,11 +232,11 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // Copy core + variant in use + preprocessed sketch in the correct folders - err := utils.CopyDir(ctx.BuildProperties.Get("build.core.path"), coreFolder.String(), validExportExtensions) + err := copyDir(ctx.BuildProperties.Get("build.core.path"), coreFolder.String(), validExportExtensions) if err != nil { fmt.Println(err) } - err = utils.CopyDir(ctx.BuildProperties.Get("build.variant.path"), coreFolder.Join("variant").String(), validExportExtensions) + err = copyDir(ctx.BuildProperties.Get("build.variant.path"), coreFolder.Join("variant").String(), validExportExtensions) if err != nil { fmt.Println(err) } @@ -119,7 +245,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { return err } - err = utils.CopyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) + err = copyDir(ctx.SketchBuildPath.String(), cmakeFolder.Join("sketch").String(), validExportExtensions) if err != nil { fmt.Println(err) } @@ -232,25 +358,34 @@ func canExportCmakeProject(ctx *types.Context) bool { } func extractCompileFlags(ctx *types.Context, recipe string, defines, dynamicLibs, linkerflags, linkDirectories *[]string) { + appendIfNotPresent := func(target []string, elements ...string) []string { + for _, element := range elements { + if !slices.Contains(target, element) { + target = append(target, element) + } + } + return target + } + command, _ := builder_utils.PrepareCommandForRecipe(ctx.BuildProperties, recipe, true, ctx.PackageManager.GetEnvVarsForSpawnedProcess()) for _, arg := range command.Args { if strings.HasPrefix(arg, "-D") { - *defines = utils.AppendIfNotPresent(*defines, arg) + *defines = appendIfNotPresent(*defines, arg) continue } if strings.HasPrefix(arg, "-l") { - *dynamicLibs = utils.AppendIfNotPresent(*dynamicLibs, arg[2:]) + *dynamicLibs = appendIfNotPresent(*dynamicLibs, arg[2:]) continue } if strings.HasPrefix(arg, "-L") { - *linkDirectories = utils.AppendIfNotPresent(*linkDirectories, arg[2:]) + *linkDirectories = appendIfNotPresent(*linkDirectories, arg[2:]) continue } if strings.HasPrefix(arg, "-") && !strings.HasPrefix(arg, "-I") && !strings.HasPrefix(arg, "-o") { // HACK : from linkerflags remove MMD (no cache is produced) if !strings.HasPrefix(arg, "-MMD") { - *linkerflags = utils.AppendIfNotPresent(*linkerflags, arg) + *linkerflags = appendIfNotPresent(*linkerflags, arg) } } } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index b05c409c488..61ff5883b65 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -19,8 +19,6 @@ import ( "bytes" "crypto/md5" "encoding/hex" - "fmt" - "io" "os" "os/exec" "path/filepath" @@ -184,15 +182,6 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (path return dir.ReadDir(fileFilter) } -func AppendIfNotPresent(target []string, elements ...string) []string { - for _, element := range elements { - if !slices.Contains(target, element) { - target = append(target, element) - } - } - return target -} - func MD5Sum(data []byte) string { md5sumBytes := md5.Sum(data) return hex.EncodeToString(md5sumBytes[:]) @@ -226,124 +215,3 @@ func NormalizeUTF8(buf []byte) []byte { result, _, _ := transform.Bytes(t, buf) return result } - -// CopyFile copies the contents of the file named src to the file named -// by dst. The file will be created if it does not already exist. If the -// destination file exists, all it's contents will be replaced by the contents -// of the source file. The file mode will be copied from the source and -// the copied data is synced/flushed to stable storage. -func CopyFile(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - if e := out.Close(); e != nil { - err = e - } - }() - - _, err = io.Copy(out, in) - if err != nil { - return - } - - err = out.Sync() - if err != nil { - return - } - - si, err := os.Stat(src) - if err != nil { - return - } - err = os.Chmod(dst, si.Mode()) - if err != nil { - return - } - - return -} - -// CopyDir recursively copies a directory tree, attempting to preserve permissions. -// Source directory must exist, destination directory must *not* exist. -// Symlinks are ignored and skipped. -func CopyDir(src string, dst string, extensions []string) (err error) { - isAcceptedExtension := func(ext string) bool { - ext = strings.ToLower(ext) - for _, valid := range extensions { - if ext == valid { - return true - } - } - return false - } - - src = filepath.Clean(src) - dst = filepath.Clean(dst) - - si, err := os.Stat(src) - if err != nil { - return err - } - if !si.IsDir() { - return fmt.Errorf(tr("source is not a directory")) - } - - _, err = os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - return - } - if err == nil { - return fmt.Errorf(tr("destination already exists")) - } - - err = os.MkdirAll(dst, si.Mode()) - if err != nil { - return - } - - entries, err := os.ReadDir(src) - if err != nil { - return - } - - for _, dirEntry := range entries { - entry, scopeErr := dirEntry.Info() - if scopeErr != nil { - return - } - - srcPath := filepath.Join(src, entry.Name()) - dstPath := filepath.Join(dst, entry.Name()) - - if entry.IsDir() { - err = CopyDir(srcPath, dstPath, extensions) - if err != nil { - return - } - } else { - // Skip symlinks. - if entry.Mode()&os.ModeSymlink != 0 { - continue - } - - if !isAcceptedExtension(filepath.Ext(srcPath)) { - continue - } - - err = CopyFile(srcPath, dstPath) - if err != nil { - return - } - } - } - - return -} From 58c91391f51a0bd71b63da93461f9e267f8720ac Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 18:46:23 +0200 Subject: [PATCH 07/15] Removed now dead code --- legacy/builder/utils/utils.go | 36 ----------------------------------- 1 file changed, 36 deletions(-) diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 61ff5883b65..9ba7a385a38 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -21,25 +21,18 @@ import ( "encoding/hex" "os" "os/exec" - "path/filepath" "strings" "unicode" - "github.com/arduino/arduino-cli/i18n" f "github.com/arduino/arduino-cli/internal/algorithms" "github.com/arduino/arduino-cli/legacy/builder/types" paths "github.com/arduino/go-paths-helper" "github.com/pkg/errors" - "golang.org/x/exp/slices" "golang.org/x/text/runes" "golang.org/x/text/transform" "golang.org/x/text/unicode/norm" ) -type filterFiles func([]os.FileInfo) []os.FileInfo - -var tr = i18n.Tr - var SOURCE_CONTROL_FOLDERS = map[string]bool{"CVS": true, "RCS": true, ".git": true, ".github": true, ".svn": true, ".hg": true, ".bzr": true, ".vscode": true, ".settings": true, ".pioenvs": true, ".piolibdeps": true} // FilterOutHiddenFiles is a ReadDirFilter that exclude files with a "." prefix in their name @@ -61,20 +54,6 @@ func FilterReadableFiles(file *paths.Path) bool { return true } -func IsSCCSOrHiddenFile(file os.FileInfo) bool { - return IsSCCSFile(file) || IsHiddenFile(file) -} - -func IsHiddenFile(file os.FileInfo) bool { - name := filepath.Base(file.Name()) - return name[0] == '.' -} - -func IsSCCSFile(file os.FileInfo) bool { - name := filepath.Base(file.Name()) - return SOURCE_CONTROL_FOLDERS[name] -} - func WrapWithHyphenI(value string) string { return "\"-I" + value + "\"" } @@ -149,21 +128,6 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) return outbytes, errbytes, errors.WithStack(err) } -func AbsolutizePaths(files []string) ([]string, error) { - for idx, file := range files { - if file == "" { - continue - } - absFile, err := filepath.Abs(file) - if err != nil { - return nil, errors.WithStack(err) - } - files[idx] = absFile - } - - return files, nil -} - func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (paths.PathList, error) { fileFilter := paths.AndFilter( paths.FilterSuffixes(extensions...), From efe545c8e20c416be5037e19228cba6c7d144294 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 25 Aug 2023 18:52:49 +0200 Subject: [PATCH 08/15] Moved md5 function to proper place --- legacy/builder/phases/core_builder.go | 13 ++++++++++--- legacy/builder/utils/utils.go | 7 ------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index e60af767b42..5883a6cb112 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -16,6 +16,8 @@ package phases import ( + "crypto/md5" + "encoding/hex" "fmt" "os" "strings" @@ -157,11 +159,16 @@ func GetCachedCoreArchiveDirName(fqbn string, optimizationFlags string, coreFold if absCoreFolder, err := coreFolder.Abs(); err == nil { coreFolder = absCoreFolder } // silently continue if absolute path can't be detected - hash := utils.MD5Sum([]byte(coreFolder.String() + optimizationFlags)) + + md5Sum := func(data []byte) string { + md5sumBytes := md5.Sum(data) + return hex.EncodeToString(md5sumBytes[:]) + } + hash := md5Sum([]byte(coreFolder.String() + optimizationFlags)) realName := fqbnToUnderscore + "_" + hash if len(realName) > 100 { - // avoid really long names, simply hash the name - realName = utils.MD5Sum([]byte(fqbnToUnderscore + "_" + hash)) + // avoid really long names, simply hash the name again + realName = md5Sum([]byte(realName)) } return realName } diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 9ba7a385a38..0c39c0efdc2 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -17,8 +17,6 @@ package utils import ( "bytes" - "crypto/md5" - "encoding/hex" "os" "os/exec" "strings" @@ -146,11 +144,6 @@ func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (path return dir.ReadDir(fileFilter) } -func MD5Sum(data []byte) string { - md5sumBytes := md5.Sum(data) - return hex.EncodeToString(md5sumBytes[:]) -} - type loggerAction struct { onlyIfVerbose bool warn bool From acae0976deded944d1494956e1275b3af528e06a Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 11:28:16 +0200 Subject: [PATCH 09/15] Refactor findAllFilesInFolder to use more go-paths function --- legacy/builder/builder_utils/utils.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index d1ab6b3bbce..517e7c87d52 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -38,17 +38,13 @@ import ( var tr = i18n.Tr -func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { - allFiles, err := paths.New(sourcePath).ReadDir() +func findAllFilesInFolder(sourcePath *paths.Path, recurse bool) (paths.PathList, error) { + allFiles, err := sourcePath.ReadDir() if err != nil { return nil, errors.WithStack(err) } - files := allFiles.Clone() - files.FilterOutDirs() - var sources []string - for _, file := range files { - sources = append(sources, filepath.Join(sourcePath, file.Base())) - } + sources := allFiles.Clone() + sources.FilterOutDirs() if recurse { folders := allFiles.Clone() @@ -59,7 +55,7 @@ func findAllFilesInFolder(sourcePath string, recurse bool) ([]string, error) { if utils.SOURCE_CONTROL_FOLDERS[folder.Base()] { continue } - otherSources, err := findAllFilesInFolder(filepath.Join(sourcePath, folder.Base()), recurse) + otherSources, err := findAllFilesInFolder(folder, recurse) if err != nil { return nil, errors.WithStack(err) } @@ -336,12 +332,12 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths. targetFileStat, err := targetFile.Stat() if err == nil { - files, err := findAllFilesInFolder(corePath.String(), true) + files, err := findAllFilesInFolder(corePath, true) if err != nil { return true } for _, file := range files { - fileStat, err := os.Stat(file) + fileStat, err := file.Stat() if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { return true } @@ -358,16 +354,16 @@ func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile *paths.Path) targetFileStat, err := targetFile.Stat() if err == nil { - files, err := findAllFilesInFolder(corePath.String(), true) + files, err := findAllFilesInFolder(corePath, true) if err != nil { return true } for _, file := range files { // report changes only for .txt files - if filepath.Ext(file) != ".txt" { + if file.Ext() != ".txt" { continue } - fileStat, err := os.Stat(file) + fileStat, err := file.Stat() if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { return true } From b1935375331403400fc78850124dd61af77dd03b Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 12:30:19 +0200 Subject: [PATCH 10/15] Refactor findAllFilesInFolder to use more go-paths function --- legacy/builder/builder_utils/utils.go | 29 +++++---------------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index 517e7c87d52..bb123aad174 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -39,31 +39,12 @@ import ( var tr = i18n.Tr func findAllFilesInFolder(sourcePath *paths.Path, recurse bool) (paths.PathList, error) { - allFiles, err := sourcePath.ReadDir() - if err != nil { - return nil, errors.WithStack(err) + if !recurse { + return sourcePath.ReadDir(paths.FilterOutDirectories()) } - sources := allFiles.Clone() - sources.FilterOutDirs() - - if recurse { - folders := allFiles.Clone() - folders.FilterOutHiddenFiles() - folders.FilterDirs() - for _, folder := range folders { - // Skip SCCS directories as they do not influence the build and can be very large - if utils.SOURCE_CONTROL_FOLDERS[folder.Base()] { - continue - } - otherSources, err := findAllFilesInFolder(folder, recurse) - if err != nil { - return nil, errors.WithStack(err) - } - sources = append(sources, otherSources...) - } - } - - return sources, nil + return sourcePath.ReadDirRecursiveFiltered( + utils.FilterOutSCCS, + paths.FilterOutDirectories()) } func CompileFiles(ctx *types.Context, sourcePath *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { From 8b7738fc834e6d9dfc199d4c705e0b1cf868e2b4 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 12:53:59 +0200 Subject: [PATCH 11/15] Made FilesAreOlderThan function to rationalize code --- legacy/builder/builder_utils/utils.go | 79 ++++++++++++++------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index bb123aad174..f324940cfdb 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -47,6 +47,25 @@ func findAllFilesInFolder(sourcePath *paths.Path, recurse bool) (paths.PathList, paths.FilterOutDirectories()) } +// FilesAreOlderThan returns true if the given files are older than target. +func FilesAreOlderThan(files paths.PathList, target *paths.Path) (bool, error) { + targetStat, err := target.Stat() + if err != nil { + return false, err + } + targetModTime := targetStat.ModTime() + for _, file := range files { + file, err := file.Stat() + if err != nil { + return false, err + } + if file.ModTime().After(targetModTime) { + return false, nil + } + } + return true, nil +} + func CompileFiles(ctx *types.Context, sourcePath *paths.Path, buildPath *paths.Path, buildProperties *properties.Map, includes []string) (paths.PathList, error) { return compileFiles(ctx, sourcePath, false, buildPath, buildProperties, includes) } @@ -310,51 +329,35 @@ func removeEndingBackSlash(s string) string { } func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool { + files, err := findAllFilesInFolder(corePath, true) + if err != nil { + return true + } - targetFileStat, err := targetFile.Stat() - if err == nil { - files, err := findAllFilesInFolder(corePath, true) - if err != nil { - return true - } - for _, file := range files { - fileStat, err := file.Stat() - if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { - return true - } - } - if targetCorePath != nil && !strings.EqualFold(corePath.String(), targetCorePath.String()) { - return CoreOrReferencedCoreHasChanged(targetCorePath, nil, targetFile) - } - return false + if isOlder, err := FilesAreOlderThan(files, targetFile); err != nil || !isOlder { + return true + } + + if targetCorePath != nil && !strings.EqualFold(corePath.String(), targetCorePath.String()) { + return CoreOrReferencedCoreHasChanged(targetCorePath, nil, targetFile) } - return true + return false } func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile *paths.Path) bool { + files, err := utils.FindFilesInFolder(corePath, true, []string{".txt"}) + if err != nil { + return true + } - targetFileStat, err := targetFile.Stat() - if err == nil { - files, err := findAllFilesInFolder(corePath, true) - if err != nil { - return true - } - for _, file := range files { - // report changes only for .txt files - if file.Ext() != ".txt" { - continue - } - fileStat, err := file.Stat() - if err != nil || fileStat.ModTime().After(targetFileStat.ModTime()) { - return true - } - } - if targetCorePath != nil && !corePath.EqualsTo(targetCorePath) { - return TXTBuildRulesHaveChanged(targetCorePath, nil, targetFile) - } - return false + if isOlder, err := FilesAreOlderThan(files, targetFile); err != nil || !isOlder { + return true + } + + if targetCorePath != nil && !corePath.EqualsTo(targetCorePath) { + return TXTBuildRulesHaveChanged(targetCorePath, nil, targetFile) } - return true + return false } func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { From e5b6e463d1fbcc577f7b42a0ac7e745b35f1d6ec Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 13:05:30 +0200 Subject: [PATCH 12/15] Yet another code-factorization of findAllFilesInFolder --- legacy/builder/builder_utils/utils.go | 13 ++----------- legacy/builder/container_find_includes.go | 2 +- legacy/builder/create_cmake_rule.go | 8 ++++---- legacy/builder/utils/utils.go | 9 +++++++-- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index f324940cfdb..af5349bf5bd 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -38,15 +38,6 @@ import ( var tr = i18n.Tr -func findAllFilesInFolder(sourcePath *paths.Path, recurse bool) (paths.PathList, error) { - if !recurse { - return sourcePath.ReadDir(paths.FilterOutDirectories()) - } - return sourcePath.ReadDirRecursiveFiltered( - utils.FilterOutSCCS, - paths.FilterOutDirectories()) -} - // FilesAreOlderThan returns true if the given files are older than target. func FilesAreOlderThan(files paths.PathList, target *paths.Path) (bool, error) { targetStat, err := target.Stat() @@ -329,7 +320,7 @@ func removeEndingBackSlash(s string) string { } func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - files, err := findAllFilesInFolder(corePath, true) + files, err := utils.FindFilesInFolder(corePath, true) if err != nil { return true } @@ -345,7 +336,7 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths. } func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - files, err := utils.FindFilesInFolder(corePath, true, []string{".txt"}) + files, err := utils.FindFilesInFolder(corePath, true, ".txt") if err != nil { return true } diff --git a/legacy/builder/container_find_includes.go b/legacy/builder/container_find_includes.go index 4c78e18ce18..5d849b927ed 100644 --- a/legacy/builder/container_find_includes.go +++ b/legacy/builder/container_find_includes.go @@ -427,7 +427,7 @@ func queueSourceFilesFromFolder(ctx *types.Context, sourceFileQueue *types.Uniqu for k := range globals.SourceFilesValidExtensions { sourceFileExtensions = append(sourceFileExtensions, k) } - filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions) + filePaths, err := utils.FindFilesInFolder(folder, recurse, sourceFileExtensions...) if err != nil { return errors.WithStack(err) } diff --git a/legacy/builder/create_cmake_rule.go b/legacy/builder/create_cmake_rule.go index 9a2bbecef60..acbce09e2cb 100644 --- a/legacy/builder/create_cmake_rule.go +++ b/legacy/builder/create_cmake_rule.go @@ -222,7 +222,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // Remove stray folders contining incompatible or not needed libraries archives - files, _ := utils.FindFilesInFolder(libDir.Join("src"), true, validStaticLibExtensions) + files, _ := utils.FindFilesInFolder(libDir.Join("src"), true, validStaticLibExtensions...) for _, file := range files { staticLibDir := file.Parent() if !isStaticLib || !strings.Contains(staticLibDir.String(), mcu) { @@ -251,7 +251,7 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { } // remove "#line 1 ..." from exported c_make folder sketch - sketchFiles, _ := utils.FindFilesInFolder(cmakeFolder.Join("sketch"), false, validExportExtensions) + sketchFiles, _ := utils.FindFilesInFolder(cmakeFolder.Join("sketch"), false, validExportExtensions...) for _, file := range sketchFiles { input, err := file.ReadFile() @@ -285,11 +285,11 @@ func (s *ExportProjectCMake) Run(ctx *types.Context) error { extractCompileFlags(ctx, "recipe.cpp.o.pattern", &defines, &dynamicLibsFromGccMinusL, &linkerflags, &linkDirectories) // Extract folders with .h in them for adding in include list - headerFiles, _ := utils.FindFilesInFolder(cmakeFolder, true, validHeaderExtensions) + headerFiles, _ := utils.FindFilesInFolder(cmakeFolder, true, validHeaderExtensions...) foldersContainingHeaders := findUniqueFoldersRelative(headerFiles.AsStrings(), cmakeFolder.String()) // Extract folders with .a in them for adding in static libs paths list - staticLibs, _ := utils.FindFilesInFolder(cmakeFolder, true, validStaticLibExtensions) + staticLibs, _ := utils.FindFilesInFolder(cmakeFolder, true, validStaticLibExtensions...) // Generate the CMakeLists global file diff --git a/legacy/builder/utils/utils.go b/legacy/builder/utils/utils.go index 0c39c0efdc2..102b007883f 100644 --- a/legacy/builder/utils/utils.go +++ b/legacy/builder/utils/utils.go @@ -126,14 +126,19 @@ func ExecCommand(ctx *types.Context, command *exec.Cmd, stdout int, stderr int) return outbytes, errbytes, errors.WithStack(err) } -func FindFilesInFolder(dir *paths.Path, recurse bool, extensions []string) (paths.PathList, error) { +func FindFilesInFolder(dir *paths.Path, recurse bool, extensions ...string) (paths.PathList, error) { fileFilter := paths.AndFilter( - paths.FilterSuffixes(extensions...), FilterOutHiddenFiles, FilterOutSCCS, paths.FilterOutDirectories(), FilterReadableFiles, ) + if len(extensions) > 0 { + fileFilter = paths.AndFilter( + paths.FilterSuffixes(extensions...), + fileFilter, + ) + } if recurse { dirFilter := paths.AndFilter( FilterOutHiddenFiles, From 123e3e495fb1c31955e445fe52bf269d2e537e54 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 16:08:06 +0200 Subject: [PATCH 13/15] Further simplified version of FilesAreOlderThan --- legacy/builder/builder_utils/utils.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index af5349bf5bd..eedd6b37cd5 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -38,13 +38,20 @@ import ( var tr = i18n.Tr -// FilesAreOlderThan returns true if the given files are older than target. -func FilesAreOlderThan(files paths.PathList, target *paths.Path) (bool, error) { +// DirContentIsOlderThan returns true if the content of the given directory is +// older than target file. If extensions are given, only the files with these +// extensions are tested. +func DirContentIsOlderThan(dir *paths.Path, target *paths.Path, extensions ...string) (bool, error) { targetStat, err := target.Stat() if err != nil { return false, err } targetModTime := targetStat.ModTime() + + files, err := utils.FindFilesInFolder(dir, true, extensions...) + if err != nil { + return false, err + } for _, file := range files { file, err := file.Stat() if err != nil { @@ -320,12 +327,7 @@ func removeEndingBackSlash(s string) string { } func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - files, err := utils.FindFilesInFolder(corePath, true) - if err != nil { - return true - } - - if isOlder, err := FilesAreOlderThan(files, targetFile); err != nil || !isOlder { + if isOlder, err := DirContentIsOlderThan(corePath, targetFile); err != nil || !isOlder { return true } @@ -336,12 +338,7 @@ func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths. } func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - files, err := utils.FindFilesInFolder(corePath, true, ".txt") - if err != nil { - return true - } - - if isOlder, err := FilesAreOlderThan(files, targetFile); err != nil || !isOlder { + if isOlder, err := DirContentIsOlderThan(corePath, targetFile, ".txt"); err != nil || !isOlder { return true } From 5178bae883989d70ae228aa3c612c8e8f0fef243 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 16:08:51 +0200 Subject: [PATCH 14/15] Inlined CoreOrReferencedCoreHasChanged function --- legacy/builder/builder_utils/utils.go | 11 ----------- legacy/builder/phases/core_builder.go | 25 ++++++++++++++++--------- 2 files changed, 16 insertions(+), 20 deletions(-) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index eedd6b37cd5..d137021230a 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -326,17 +326,6 @@ func removeEndingBackSlash(s string) string { return strings.TrimSuffix(s, "\\") } -func CoreOrReferencedCoreHasChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - if isOlder, err := DirContentIsOlderThan(corePath, targetFile); err != nil || !isOlder { - return true - } - - if targetCorePath != nil && !strings.EqualFold(corePath.String(), targetCorePath.String()) { - return CoreOrReferencedCoreHasChanged(targetCorePath, nil, targetFile) - } - return false -} - func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile *paths.Path) bool { if isOlder, err := DirContentIsOlderThan(corePath, targetFile, ".txt"); err != nil || !isOlder { return true diff --git a/legacy/builder/phases/core_builder.go b/legacy/builder/phases/core_builder.go index 5883a6cb112..5c94573f349 100644 --- a/legacy/builder/phases/core_builder.go +++ b/legacy/builder/phases/core_builder.go @@ -92,26 +92,33 @@ func compileCore(ctx *types.Context, buildPath *paths.Path, buildCachePath *path } } - // Recreate the archive if ANY of the core files (including platform.txt) has changed - realCoreFolder := coreFolder.Parent().Parent() - var targetArchivedCore *paths.Path - var buildCacheErr error if buildCachePath != nil { + realCoreFolder := coreFolder.Parent().Parent() archivedCoreName := GetCachedCoreArchiveDirName( buildProperties.Get("build.fqbn"), buildProperties.Get("compiler.optimization_flags"), realCoreFolder) targetArchivedCore = buildCachePath.Join(archivedCoreName, "core.a") - _, buildCacheErr = buildcache.New(buildCachePath).GetOrCreate(archivedCoreName) - if errors.Is(buildCacheErr, buildcache.CreateDirErr) { + if _, err := buildcache.New(buildCachePath).GetOrCreate(archivedCoreName); errors.Is(err, buildcache.CreateDirErr) { return nil, nil, fmt.Errorf(tr("creating core cache folder: %s", err)) } - canUseArchivedCore := !ctx.OnlyUpdateCompilationDatabase && - !ctx.Clean && - !builder_utils.CoreOrReferencedCoreHasChanged(realCoreFolder, targetCoreFolder, targetArchivedCore) + var canUseArchivedCore bool + if ctx.OnlyUpdateCompilationDatabase || ctx.Clean { + canUseArchivedCore = false + } else if isOlder, err := builder_utils.DirContentIsOlderThan(realCoreFolder, targetArchivedCore); err != nil || !isOlder { + // Recreate the archive if ANY of the core files (including platform.txt) has changed + canUseArchivedCore = false + } else if targetCoreFolder == nil || realCoreFolder.EquivalentTo(targetCoreFolder) { + canUseArchivedCore = true + } else if isOlder, err := builder_utils.DirContentIsOlderThan(targetCoreFolder, targetArchivedCore); err != nil || !isOlder { + // Recreate the archive if ANY of the build core files (including platform.txt) has changed + canUseArchivedCore = false + } else { + canUseArchivedCore = true + } if canUseArchivedCore { // use archived core From 22dcc9e3d6e12a344acbb97889c6f97ada29fde1 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Mon, 28 Aug 2023 16:50:28 +0200 Subject: [PATCH 15/15] Inlined TXTBuildRulesHaveChanged function --- legacy/builder/builder_utils/utils.go | 11 ----------- .../wipeout_build_path_if_build_options_changed.go | 8 +++++--- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/legacy/builder/builder_utils/utils.go b/legacy/builder/builder_utils/utils.go index d137021230a..4a227e76154 100644 --- a/legacy/builder/builder_utils/utils.go +++ b/legacy/builder/builder_utils/utils.go @@ -326,17 +326,6 @@ func removeEndingBackSlash(s string) string { return strings.TrimSuffix(s, "\\") } -func TXTBuildRulesHaveChanged(corePath, targetCorePath, targetFile *paths.Path) bool { - if isOlder, err := DirContentIsOlderThan(corePath, targetFile, ".txt"); err != nil || !isOlder { - return true - } - - if targetCorePath != nil && !corePath.EqualsTo(targetCorePath) { - return TXTBuildRulesHaveChanged(targetCorePath, nil, targetFile) - } - return false -} - func ArchiveCompiledFiles(ctx *types.Context, buildPath *paths.Path, archiveFile *paths.Path, objectFilesToArchive paths.PathList, buildProperties *properties.Map) (*paths.Path, error) { archiveFilePath := buildPath.JoinPath(archiveFile) diff --git a/legacy/builder/wipeout_build_path_if_build_options_changed.go b/legacy/builder/wipeout_build_path_if_build_options_changed.go index 44e5207a619..2a6873de97b 100644 --- a/legacy/builder/wipeout_build_path_if_build_options_changed.go +++ b/legacy/builder/wipeout_build_path_if_build_options_changed.go @@ -66,9 +66,11 @@ func (s *WipeoutBuildPathIfBuildOptionsChanged) Run(ctx *types.Context) error { coreFolder := buildProperties.GetPath("build.core.path") realCoreFolder := coreFolder.Parent().Parent() jsonPath := ctx.BuildPath.Join(constants.BUILD_OPTIONS_FILE) - coreHasChanged := builder_utils.TXTBuildRulesHaveChanged(realCoreFolder, targetCoreFolder, jsonPath) - - if !coreHasChanged { + coreUnchanged, _ := builder_utils.DirContentIsOlderThan(realCoreFolder, jsonPath, ".txt") + if coreUnchanged && targetCoreFolder != nil && !realCoreFolder.EqualsTo(targetCoreFolder) { + coreUnchanged, _ = builder_utils.DirContentIsOlderThan(targetCoreFolder, jsonPath, ".txt") + } + if coreUnchanged { return nil } }