mirror of https://github.com/facebook/rocksdb.git
List blob files when using command - list_live_files_metadata (#8976)
Summary: The ldb list_live_files_metadata command does not print any information about blob files currently. We would like to add this functionality. Note that list_live_files_metadata has two different modes of operation: the one shown above, which shows the LSM tree structure, and another one, which can be enabled using the flag --sort_by_filename and simply lists the files in numerical order regardless of level. We would like to show blob files in both modes. Changes: 1. Using GetAllColumnFamilyMetaData API instead of GetLiveFilesMetaData API for fetching live files data. Testing: 1. Created a sample rocksdb instance using dbbench command (this creates both SST and blob files) 2. Checked if the blob files are listed or not by using ldb commands. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8976 Reviewed By: ltamasi Differential Revision: D31316061 Pulled By: pradeepambati fbshipit-source-id: d15cdea192febf7a45f28deee2ba40615d3d84ab
This commit is contained in:
parent
1953b63cdd
commit
e5bfb91d09
|
@ -5,6 +5,7 @@
|
|||
* Fix the incorrect disabling of SST rate limited deletion when the WAL and DB are in different directories. Only WAL rate limited deletion should be disabled if its in a different directory.
|
||||
|
||||
### New Features
|
||||
* Print information about blob files when using "ldb list_live_files_metadata"
|
||||
* Provided support for SingleDelete with user defined timestamp.
|
||||
* Add remote compaction read/write bytes statistics: `REMOTE_COMPACT_READ_BYTES`, `REMOTE_COMPACT_WRITE_BYTES`.
|
||||
|
||||
|
|
165
tools/ldb_cmd.cc
165
tools/ldb_cmd.cc
|
@ -3445,89 +3445,100 @@ void DBLiveFilesMetadataDumperCommand::DoCommand() {
|
|||
}
|
||||
Status s;
|
||||
|
||||
std::cout << "Live SST Files:" << std::endl;
|
||||
std::vector<LiveFileMetaData> metadata;
|
||||
db_->GetLiveFilesMetaData(&metadata);
|
||||
std::vector<ColumnFamilyMetaData> metadata;
|
||||
db_->GetAllColumnFamilyMetaData(&metadata);
|
||||
if (sort_by_filename_) {
|
||||
// Sort metadata vector by filename.
|
||||
std::sort(metadata.begin(), metadata.end(),
|
||||
[](const LiveFileMetaData& a, const LiveFileMetaData& b) -> bool {
|
||||
std::string aName = a.db_path + a.name;
|
||||
std::string bName = b.db_path + b.name;
|
||||
return (aName.compare(bName) < 0);
|
||||
});
|
||||
for (auto& fileMetadata : metadata) {
|
||||
// The fileMetada.name alwasy starts with "/",
|
||||
// however fileMetada.db_path is the string provided by
|
||||
// the user as an input. Therefore we check if we can
|
||||
// concantenate the two string sdirectly or if we need to
|
||||
// drop a possible extra "/" at the end of fileMetadata.db_path.
|
||||
std::string filename = fileMetadata.db_path + "/" + fileMetadata.name;
|
||||
// Drops any repeating '/' character that could happen during
|
||||
// concatenation of db path and file name.
|
||||
filename = NormalizePath(filename);
|
||||
std::string cf = fileMetadata.column_family_name;
|
||||
int level = fileMetadata.level;
|
||||
std::cout << filename << " : level " << level << ", column family '" << cf
|
||||
<< "'" << std::endl;
|
||||
std::cout << "Live SST and Blob Files:" << std::endl;
|
||||
// tuple of <file path, level, column family name>
|
||||
std::vector<std::tuple<std::string, int, std::string>> all_files;
|
||||
|
||||
for (const auto& column_metadata : metadata) {
|
||||
// Iterate Levels
|
||||
const auto& levels = column_metadata.levels;
|
||||
const std::string& cf = column_metadata.name;
|
||||
for (const auto& level_metadata : levels) {
|
||||
// Iterate SST files
|
||||
const auto& sst_files = level_metadata.files;
|
||||
int level = level_metadata.level;
|
||||
for (const auto& sst_metadata : sst_files) {
|
||||
// The SstFileMetaData.name always starts with "/",
|
||||
// however SstFileMetaData.db_path is the string provided by
|
||||
// the user as an input. Therefore we check if we can
|
||||
// concantenate the two strings directly or if we need to
|
||||
// drop a possible extra "/" at the end of SstFileMetaData.db_path.
|
||||
std::string filename =
|
||||
NormalizePath(sst_metadata.db_path + "/" + sst_metadata.name);
|
||||
all_files.emplace_back(filename, level, cf);
|
||||
} // End of for-loop over sst files
|
||||
} // End of for-loop over levels
|
||||
|
||||
const auto& blob_files = column_metadata.blob_files;
|
||||
for (const auto& blob_metadata : blob_files) {
|
||||
// The BlobMetaData.blob_file_name always starts with "/",
|
||||
// however BlobMetaData.blob_file_path is the string provided by
|
||||
// the user as an input. Therefore we check if we can
|
||||
// concantenate the two strings directly or if we need to
|
||||
// drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
|
||||
std::string filename = NormalizePath(
|
||||
blob_metadata.blob_file_path + "/" + blob_metadata.blob_file_name);
|
||||
// Level for blob files is encoded as -1
|
||||
all_files.emplace_back(filename, -1, cf);
|
||||
} // End of for-loop over blob files
|
||||
} // End of for-loop over column metadata
|
||||
|
||||
// Sort by filename (i.e. first entry in tuple)
|
||||
std::sort(all_files.begin(), all_files.end());
|
||||
|
||||
for (const auto& item : all_files) {
|
||||
const std::string& filename = std::get<0>(item);
|
||||
int level = std::get<1>(item);
|
||||
const std::string& cf = std::get<2>(item);
|
||||
if (level == -1) { // Blob File
|
||||
std::cout << filename << ", column family '" << cf << "'" << std::endl;
|
||||
} else { // SST file
|
||||
std::cout << filename << " : level " << level << ", column family '"
|
||||
<< cf << "'" << std::endl;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
std::map<std::string, std::map<int, std::vector<std::string>>>
|
||||
filesPerLevelPerCf;
|
||||
// Collect live files metadata.
|
||||
// Store filenames into a 2D map, that will automatically
|
||||
// sort by column family (first key) and by level (second key).
|
||||
for (auto& fileMetadata : metadata) {
|
||||
std::string cf = fileMetadata.column_family_name;
|
||||
int level = fileMetadata.level;
|
||||
if (filesPerLevelPerCf.find(cf) == filesPerLevelPerCf.end()) {
|
||||
filesPerLevelPerCf.emplace(cf,
|
||||
std::map<int, std::vector<std::string>>());
|
||||
}
|
||||
if (filesPerLevelPerCf[cf].find(level) == filesPerLevelPerCf[cf].end()) {
|
||||
filesPerLevelPerCf[cf].emplace(level, std::vector<std::string>());
|
||||
}
|
||||
|
||||
// The fileMetada.name alwasy starts with "/",
|
||||
// however fileMetada.db_path is the string provided by
|
||||
// the user as an input. Therefore we check if we can
|
||||
// concantenate the two string sdirectly or if we need to
|
||||
// drop a possible extra "/" at the end of fileMetadata.db_path.
|
||||
std::string filename = fileMetadata.db_path + "/" + fileMetadata.name;
|
||||
// Drops any repeating '/' character that could happen during
|
||||
// concatenation of db path and file name.
|
||||
filename = NormalizePath(filename);
|
||||
filesPerLevelPerCf[cf][level].push_back(filename);
|
||||
}
|
||||
// For each column family,
|
||||
// iterate through the levels and print out the live SST file names.
|
||||
for (auto it = filesPerLevelPerCf.begin(); it != filesPerLevelPerCf.end();
|
||||
it++) {
|
||||
// it->first: Column Family name (string)
|
||||
// it->second: map[level]={SST files...}.
|
||||
std::cout << "===== Column Family: " << it->first
|
||||
for (const auto& column_metadata : metadata) {
|
||||
std::cout << "===== Column Family: " << column_metadata.name
|
||||
<< " =====" << std::endl;
|
||||
|
||||
// For simplicity, create reference to the inner map (level={live SST
|
||||
// files}).
|
||||
std::map<int, std::vector<std::string>>& filesPerLevel = it->second;
|
||||
int maxLevel = filesPerLevel.rbegin()->first;
|
||||
std::cout << "Live SST Files:" << std::endl;
|
||||
// Iterate levels
|
||||
const auto& levels = column_metadata.levels;
|
||||
for (const auto& level_metadata : levels) {
|
||||
std::cout << "---------- level " << level_metadata.level
|
||||
<< " ----------" << std::endl;
|
||||
// Iterate SST files
|
||||
const auto& sst_files = level_metadata.files;
|
||||
for (const auto& sst_metadata : sst_files) {
|
||||
// The SstFileMetaData.name always starts with "/",
|
||||
// however SstFileMetaData.db_path is the string provided by
|
||||
// the user as an input. Therefore we check if we can
|
||||
// concantenate the two strings directly or if we need to
|
||||
// drop a possible extra "/" at the end of SstFileMetaData.db_path.
|
||||
std::string filename =
|
||||
NormalizePath(sst_metadata.db_path + "/" + sst_metadata.name);
|
||||
std::cout << filename << std::endl;
|
||||
} // End of for-loop over sst files
|
||||
} // End of for-loop over levels
|
||||
|
||||
// Even if the first few levels are empty, they are printed out.
|
||||
for (int level = 0; level <= maxLevel; level++) {
|
||||
std::cout << "---------- level " << level << " ----------" << std::endl;
|
||||
if (filesPerLevel.find(level) != filesPerLevel.end()) {
|
||||
std::vector<std::string>& fileList = filesPerLevel[level];
|
||||
|
||||
// Locally sort by filename for better information display.
|
||||
std::sort(fileList.begin(), fileList.end());
|
||||
for (const std::string& filename : fileList) {
|
||||
std::cout << filename << std::endl;
|
||||
}
|
||||
}
|
||||
} // End of for-loop over levels.
|
||||
} // End of for-loop over filesPerLevelPerCf.
|
||||
} // End of else ("not sort_by_filename").
|
||||
std::cout << "Live Blob Files:" << std::endl;
|
||||
const auto& blob_files = column_metadata.blob_files;
|
||||
for (const auto& blob_metadata : blob_files) {
|
||||
// The BlobMetaData.blob_file_name always starts with "/",
|
||||
// however BlobMetaData.blob_file_path is the string provided by
|
||||
// the user as an input. Therefore we check if we can
|
||||
// concantenate the two strings directly or if we need to
|
||||
// drop a possible extra "/" at the end of BlobMetaData.blob_file_path.
|
||||
std::string filename = NormalizePath(
|
||||
blob_metadata.blob_file_path + "/" + blob_metadata.blob_file_name);
|
||||
std::cout << filename << std::endl;
|
||||
} // End of for-loop over blob files
|
||||
} // End of for-loop over column metadata
|
||||
} // End of else ("not sort_by_filename")
|
||||
std::cout << "------------------------------" << std::endl;
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue