From 1adebdc7676e44c91daeee24745a63c2019e2ec3 Mon Sep 17 00:00:00 2001 From: def Date: Tue, 24 Feb 2015 16:37:54 +0100 Subject: [PATCH 1/4] Speed up walkDir significantly We only know that this works on Linux and Mac OS X, so other systems use the POSIX conforming version still. This removed the lstat call, which is especially expensive on NFS filesystems for me. --- lib/posix/posix.nim | 13 +++++++++++++ lib/pure/os.nim | 10 +++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 8454967562..895e4b1d58 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -70,6 +70,16 @@ const STDIN_FILENO* = 0 ## File number of stdin; STDOUT_FILENO* = 1 ## File number of stdout; + DT_UNKNOWN* = 0 + DT_FIFO* = 1 + DT_CHR* = 2 + DT_DIR* = 4 + DT_BLK* = 6 + DT_REG* = 8 + DT_LNK* = 10 + DT_SOCK* = 12 + DT_WHT* = 14 + type TDIR* {.importc: "DIR", header: "", incompleteStruct.} = object @@ -84,6 +94,9 @@ type Tdirent* {.importc: "struct dirent", header: "", final, pure.} = object ## dirent_t struct d_ino*: Tino ## File serial number. + d_off*: TOff + d_reclen*: cshort + d_type*: int8 d_name*: array [0..255, char] ## Name of entry. Tflock* {.importc: "struct flock", final, pure, diff --git a/lib/pure/os.nim b/lib/pure/os.nim index ceeba182fe..23e01a7d4d 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1299,10 +1299,14 @@ iterator walkDir*(dir: string): tuple[kind: PathComponent, path: string] {. if y != "." and y != "..": var s: TStat y = dir / y - if lstat(y, s) < 0'i32: break var k = pcFile - if S_ISDIR(s.st_mode): k = pcDir - if S_ISLNK(s.st_mode): k = succ(k) + when defined(linux) or defined(macosx): + if x.d_type == DT_DIR: k = pcDir + if x.d_type == DT_LNK: k = succ(k) + else: + if lstat(y, s) < 0'i32: break + if S_ISDIR(s.st_mode): k = pcDir + if S_ISLNK(s.st_mode): k = succ(k) yield (k, y) discard closedir(d) From 18dd5e196508818f47fcf069479089ccb46f7eea Mon Sep 17 00:00:00 2001 From: def Date: Tue, 24 Feb 2015 16:52:01 +0100 Subject: [PATCH 2/4] Add some posix dirent documentation --- lib/posix/posix.nim | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim index 895e4b1d58..7d3e3ddbaa 100644 --- a/lib/posix/posix.nim +++ b/lib/posix/posix.nim @@ -70,14 +70,14 @@ const STDIN_FILENO* = 0 ## File number of stdin; STDOUT_FILENO* = 1 ## File number of stdout; - DT_UNKNOWN* = 0 - DT_FIFO* = 1 - DT_CHR* = 2 - DT_DIR* = 4 - DT_BLK* = 6 - DT_REG* = 8 - DT_LNK* = 10 - DT_SOCK* = 12 + DT_UNKNOWN* = 0 ## Unknown file type. + DT_FIFO* = 1 ## Named pipe, or FIFO. + DT_CHR* = 2 ## Character device. + DT_DIR* = 4 ## Directory. + DT_BLK* = 6 ## Block device. + DT_REG* = 8 ## Regular file. + DT_LNK* = 10 ## Symbolic link. + DT_SOCK* = 12 ## UNIX domain socket. DT_WHT* = 14 type @@ -94,9 +94,10 @@ type Tdirent* {.importc: "struct dirent", header: "", final, pure.} = object ## dirent_t struct d_ino*: Tino ## File serial number. - d_off*: TOff - d_reclen*: cshort - d_type*: int8 + d_off*: TOff ## Not an offset. Value that ``telldir()`` would return. + d_reclen*: cshort ## Length of this record. (not POSIX) + d_type*: int8 ## Type of file; not supported by all filesystem types. + ## (not POSIX) d_name*: array [0..255, char] ## Name of entry. Tflock* {.importc: "struct flock", final, pure, From 34d87c105cdb667b40df6c7c3c953d0c9cd6180a Mon Sep 17 00:00:00 2001 From: def Date: Tue, 24 Feb 2015 23:31:31 +0100 Subject: [PATCH 3/4] Fall back to lstat() calls on unsupported filesystems --- lib/pure/os.nim | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 23e01a7d4d..f668e756db 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1300,13 +1300,17 @@ iterator walkDir*(dir: string): tuple[kind: PathComponent, path: string] {. var s: TStat y = dir / y var k = pcFile + when defined(linux) or defined(macosx): - if x.d_type == DT_DIR: k = pcDir - if x.d_type == DT_LNK: k = succ(k) - else: - if lstat(y, s) < 0'i32: break - if S_ISDIR(s.st_mode): k = pcDir - if S_ISLNK(s.st_mode): k = succ(k) + if x.d_type != DT_UNKNOWN: + if x.d_type == DT_DIR: k = pcDir + if x.d_type == DT_LNK: k = succ(k) + yield (k, y) + continue + + if lstat(y, s) < 0'i32: break + if S_ISDIR(s.st_mode): k = pcDir + if S_ISLNK(s.st_mode): k = succ(k) yield (k, y) discard closedir(d) From 5e8eaa5f9755a6c8ed73ae0e49d399ac2dd3217c Mon Sep 17 00:00:00 2001 From: def Date: Tue, 24 Feb 2015 23:39:08 +0100 Subject: [PATCH 4/4] Use faster walkDir on BSDs --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index f668e756db..bf581667bd 100644 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1301,7 +1301,7 @@ iterator walkDir*(dir: string): tuple[kind: PathComponent, path: string] {. y = dir / y var k = pcFile - when defined(linux) or defined(macosx): + when defined(linux) or defined(macosx) or defined(bsd): if x.d_type != DT_UNKNOWN: if x.d_type == DT_DIR: k = pcDir if x.d_type == DT_LNK: k = succ(k)