Merge pull request #722 from cyberkm/master

Faster collection of files on huge codebases (posix system)

cli/filelister.cpp

@@ -162,130 +162,89 @@ bool FileLister::fileExists(const std::string &path)
 #undef __STRICT_ANSI__
 #endif
 
-#include <glob.h>
+#include <dirent.h>
 #include <unistd.h>
 #include <stdlib.h>
 #include <limits.h>
 #include <sys/stat.h>
-#include <sstream>
 
-// Get absolute path. Returns empty string if path does not exist or other error.
+
-std::string FileLister::getAbsolutePath(const std::string& path)
+static void addFiles2(std::map<std::string, std::size_t> &files,
+                      const std::string &path,
+                      const std::set<std::string> &extra,
+                      bool recursive,
+                      const PathMatch& ignored
+                     )
 {
-    std::string absolute_path;
+    struct stat file_stat;
+    if (stat(path.c_str(), &file_stat) != -1) {
+        if ((file_stat.st_mode & S_IFMT) == S_IFDIR) {
+            DIR * dir = opendir(path.c_str());
 
-#ifdef PATH_MAX
+            if (!dir)
-    char buf[PATH_MAX];
+                return;
-    if (realpath(path.c_str(), buf) != nullptr)
-        absolute_path = buf;
-#else
-    char *dynamic_buf;
-    if ((dynamic_buf = realpath(path.c_str(), nullptr)) != nullptr) {
-        absolute_path = dynamic_buf;
-        free(dynamic_buf);
-    }
-#endif
 
-    return absolute_path;
+            dirent entry;
-}
+            dirent * dir_result;
 
-void FileLister::addFiles2(std::set<std::string> &seen_paths,
+            std::string new_path;
-                           std::map<std::string, std::size_t> &files,
+            new_path.reserve(path.length() + 100);// prealloc some memory to avoid constant new/deletes in loop
-                           const std::string &path,
-                           const std::set<std::string> &extra,
-                           bool recursive,
-                           const PathMatch& ignored
-                          )
-{
-    std::ostringstream oss;
-    oss << path;
-    if (path.length() > 0 && path.back() == '/')
-        oss << "*";
 
-    glob_t glob_results;
+            while ((readdir_r(dir, &entry, &dir_result) == 0) && (dir_result != NULL)) {
-    glob(oss.str().c_str(), GLOB_MARK, 0, &glob_results);
-    for (unsigned int i = 0; i < glob_results.gl_pathc; i++) {
-        const std::string filename = glob_results.gl_pathv[i];
-        if (filename == "." || filename == ".." || filename.length() == 0)
-            continue;
 
-        // Determine absolute path. Empty filename if path does not exist
+                if ((std::strcmp(dir_result->d_name, ".") == 0) ||
-        const std::string absolute_path = getAbsolutePath(filename);
+                    (std::strcmp(dir_result->d_name, "..") == 0))
-        if (absolute_path.empty())
+                    continue;
-            continue;
 
-        // Did we already process this entry?
+                new_path = path + "/" + dir_result->d_name;
-        if (seen_paths.find(absolute_path) != seen_paths.end())
-            continue;
 
-        if (filename.back() != '/') {
+                if (dir_result->d_type == DT_DIR) {
-            // File
+                    if (recursive && !ignored.Match(new_path)) {
-
+                        addFiles2(files, new_path, extra, recursive, ignored);
-            if ((Path::sameFileName(path,filename) || Path::acceptFile(filename, extra)) && !ignored.Match(filename)) {
+                    }
-                seen_paths.insert(absolute_path);
+                } else {
-
+                    if (Path::acceptFile(new_path, extra) && !ignored.Match(new_path)) {
-                struct stat sb;
+                        files[new_path] = file_stat.st_size;
-                if (stat(absolute_path.c_str(), &sb) == 0) {
+                    }
-                    // Limitation: file sizes are assumed to fit in a 'size_t'
+                }
-                    files[filename] = static_cast<std::size_t>(sb.st_size);
-                } else
-                    files[filename] = 0;
             }
-        } else if (recursive) {
+            closedir(dir);
-            // Directory
+        } else
-            if (!ignored.Match(filename)) {
+            files[path] = file_stat.st_size;
-                seen_paths.insert(absolute_path);
-                addFiles2(seen_paths, files, filename, extra, recursive, ignored);
-            }
-        }
     }
-    globfree(&glob_results);
 }
 
-
 void FileLister::recursiveAddFiles(std::map<std::string, std::size_t> &files, const std::string &path, const std::set<std::string> &extra, const PathMatch& ignored)
 {
-    std::set<std::string> seen_paths;
+    addFiles(files, path, extra, true, ignored);
-    addFiles2(seen_paths, files, path, extra, true, ignored);
 }
 
 void FileLister::addFiles(std::map<std::string, std::size_t> &files, const std::string &path, const std::set<std::string> &extra, bool recursive, const PathMatch& ignored)
 {
-    std::set<std::string> seen_paths;
+    if (!path.empty()) {
-    addFiles2(seen_paths, files, path, extra, recursive, ignored);
+        std::string corrected_path = path;
+        if (corrected_path.back() == '/')
+            corrected_path.erase(corrected_path.end() - 1);
+
+        addFiles2(files, corrected_path, extra, recursive, ignored);
+    }
 }
 
 bool FileLister::isDirectory(const std::string &path)
 {
-    bool ret = false;
+    struct stat file_stat;
-
+    if (stat(path.c_str(), &file_stat) != -1)
-    glob_t glob_results;
+        return ((file_stat.st_mode & S_IFMT) == S_IFDIR);
-    glob(path.c_str(), GLOB_MARK, 0, &glob_results);
+    
-    if (glob_results.gl_pathc == 1) {
+    return false;
-        const std::string glob_path = glob_results.gl_pathv[0];
-        if (!glob_path.empty() && glob_path[glob_path.size() - 1] == '/') {
-            ret = true;
-        }
-    }
-    globfree(&glob_results);
-
-    return ret;
 }
 
 bool FileLister::fileExists(const std::string &path)
 {
-    struct stat statinfo;
+    struct stat file_stat;
-    int result = stat(path.c_str(), &statinfo);
+    if (stat(path.c_str(), &file_stat) != -1)
-
+        return ((file_stat.st_mode & S_IFMT) == S_IFREG);
-    if (result < 0) { // Todo: should check errno == ENOENT?
+    
-        // File not found
-        return false;
-    }
-
-    // Check if file is regular file
-    if ((statinfo.st_mode & S_IFMT) == S_IFREG)
-        return true;
-
     return false;
 }
 

cli/filelister.h

@@ -81,20 +81,6 @@ public:
       * @return true if path points to file and the file exists.
       */
     static bool fileExists(const std::string &path);
-
-#ifndef _WIN32
-    static std::string getAbsolutePath(const std::string& path);
-
-private:
-
-    static void addFiles2(std::set<std::string> &seen_paths,
-                          std::map<std::string, std::size_t> &files,
-                          const std::string &path,
-                          const std::set<std::string> &extra,
-                          bool recursive,
-                          const PathMatch& ignored);
-#endif
-
 };
 
 /// @}

test/testfilelister.cpp

@@ -46,9 +46,6 @@ private:
         }
 
         TEST_CASE(isDirectory);
-#ifndef _WIN32
-        TEST_CASE(absolutePath);
-#endif
         TEST_CASE(recursiveAddFiles);
     }
 
@@ -57,23 +54,6 @@ private:
         ASSERT_EQUALS(true, FileLister::isDirectory("lib"));
     }
 
-#ifndef _WIN32
-    void absolutePath() const {
-        std::vector<char> current_dir;
-#ifdef PATH_MAX
-        current_dir.resize(PATH_MAX);
-#else
-        current_dir.resize(1024);
-#endif
-        while (getcwd(&current_dir[0], current_dir.size()) == nullptr && errno == ERANGE) {
-            current_dir.resize(current_dir.size() + 1024);
-        }
-
-        std::string absolute_path = FileLister::getAbsolutePath(".");
-        ASSERT_EQUALS(&current_dir[0], absolute_path);
-    }
-#endif
-
     void recursiveAddFiles() const {
         // Recursively add add files..
         std::map<std::string, std::size_t> files;