Coverage for datakiln/metadata/files.py: 49%

53 statements  

« prev     ^ index     » next       coverage.py v7.17.3, created at 2026-11-16 23:23 +0000

1""" 

2Data files of the tables. 

3""" 

4from hashlib import md5 

5from pathlib import Path 

6from typing import List 

7 

8from dvc.api import DVCFileSystem 

9 

10 

11class LocalFile(dict): 

12 """Local file or folder, with the keys of one that DVC tracks. Its 

13 checksum is computed when it is needed, as that takes reading it.""" 

14 

15 def __init__(self, path: Path, repo: 'LocalRepo'): 

16 super().__init__(name=path.name, repo=repo) 

17 self._path = path 

18 

19 def __missing__(self, key): 

20 if key != "md5": 

21 raise KeyError(key) 

22 

23 # A folder is told by the names and the content of its files 

24 files = [self._path] 

25 if self._path.is_dir(): 

26 files = sorted(self._path.rglob("*")) 

27 

28 checksum = md5() # noqa: DUO130 

29 for file in files: 

30 if file.is_dir(): 

31 continue 

32 if self._path.is_dir(): 

33 checksum.update(str(file.relative_to(self._path)).encode()) 

34 with open(file, "rb") as content: 

35 while chunk := content.read(4096): 

36 checksum.update(chunk) 

37 

38 self[key] = checksum.hexdigest() 

39 return self[key] 

40 

41 

42class LocalRepo: 

43 """Folder of data files that DVC does not track. It has the part of a DVC 

44 repository, and of its file system, that the tables use, so that there is 

45 nothing to fetch.""" 

46 

47 def __init__(self, root_dir: Path): 

48 self.root_dir = str(root_dir) 

49 

50 def listdir(self, path: str, dvc_only: bool = False) -> List[LocalFile]: 

51 # pylint: disable=unused-argument 

52 """Files and folders, other than the hidden ones and the ones of DVC. 

53 

54 :param path: Folder in the root. 

55 :type path: str 

56 

57 :param dvc_only: Ignored, as every file is local. 

58 :type dvc_only: bool 

59 

60 :rtype: list[LocalFile] 

61 """ 

62 files = [] 

63 for file in sorted((Path(self.root_dir) / path).iterdir()): 

64 if not file.name.startswith(".") and file.suffix != ".dvc": 

65 files.append(LocalFile(file, self)) 

66 return files 

67 

68 def pull(self, targets): 

69 # pylint: disable=unused-argument 

70 """Nothing is fetched, since the files are local. 

71 

72 :return: What was fetched, which is nothing. 

73 :rtype: dict 

74 """ 

75 return {} 

76 

77 

78class DataFiles: 

79 # pylint: disable=too-few-public-methods 

80 """Data files of the tables. They are the ones DVC tracks if the folder is 

81 a DVC repository, and the ones in it otherwise.""" 

82 

83 def __init__(self, directory: Path): 

84 """ 

85 :param directory: Folder of the data files. 

86 :type directory: Path 

87 """ 

88 directory = Path(directory) 

89 if (directory / ".dvc").is_dir(): 

90 self._files = DVCFileSystem(directory) 

91 else: 

92 self._files = LocalRepo(directory) 

93 

94 @staticmethod 

95 def _named(path: str, name: str) -> bool: 

96 """Whether a file is named so, without its suffixes and ignoring the 

97 case.""" 

98 path = Path(path) 

99 suffix = "".join(path.suffixes) 

100 stem = path.name[:-len(suffix)] if suffix else path.name 

101 return stem.lower() == name 

102 

103 def find(self, name: str) -> dict: 

104 """Find the data of a table. 

105 

106 :param name: Name of the table metadata, in lower case. 

107 :type name: str 

108 

109 :return: Name of the file or folder in the folder of the data files, 

110 its checksum and the repository it belongs to. 

111 :rtype: dict 

112 

113 :raises FileNotFoundError: If there is no file for the table. 

114 """ 

115 for artifact in self._files.listdir(".", dvc_only=True): 

116 if self._named(artifact["name"], name): 

117 return artifact 

118 

119 raise FileNotFoundError(f"No files found for metadata: '{name}'")