From 3fa73f36694f27bc686dd85e1c0b30bacc27f9b4 Mon Sep 17 00:00:00 2001 From: duqian Date: Fri, 3 May 2024 15:36:22 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=91=E7=9B=98dentryfile?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=5F2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: duqian --- utils/dentry/src/meta_file_clouddisk.cpp | 394 +++++++++++++++++++++++ 1 file changed, 394 insertions(+) diff --git a/utils/dentry/src/meta_file_clouddisk.cpp b/utils/dentry/src/meta_file_clouddisk.cpp index e183d579..a6377a3f 100644 --- a/utils/dentry/src/meta_file_clouddisk.cpp +++ b/utils/dentry/src/meta_file_clouddisk.cpp @@ -38,6 +38,12 @@ constexpr uint32_t DENTRY_PER_GROUP = 52; constexpr uint32_t DENTRY_BITMAP_LENGTH = 7; constexpr uint32_t DENTRY_GROUP_RESERVED = 32; constexpr uint32_t CLOUD_RECORD_ID_LEN = 33; +constexpr uint32_t DENTRYGROUP_HEADER = 4096; +constexpr uint32_t MAX_BUCKET_LEVEL = 63; +constexpr uint32_t BUCKET_BLOCKS = 2; +constexpr uint32_t BITS_PER_BYTE = 8; +constexpr uint32_t HMDFS_SLOT_LEN_BITS = 3; +constexpr uint64_t DELTA = 0x9E3779B9; /* Hashing code copied from f2fs */ constexpr uint32_t CLOUD_ID_BUCKET_MAX_SIZE = 32; constexpr uint32_t CLOUD_ID_BUCKET_MID_TIMES = 2; constexpr uint32_t CLOUD_ID_MIN_SIZE = 3; @@ -154,8 +160,214 @@ CloudDiskMetaFile::~CloudDiskMetaFile() (void)FileUtils::WriteFile(fd_, &header, 0, sizeof(header)); } +static bool IsDotDotdot(const std::string &name) +{ + return name == "." || name == ".."; +} + +static void Str2HashBuf(const char *msg, size_t len, uint32_t *buf, int num) +{ + uint32_t pad = static_cast(len) | (static_cast(len) << 8); + pad |= pad << 16; /* hash pad length 16 */ + + uint32_t val = pad; + len = std::min(len, static_cast(num * sizeof(int))); + for (uint32_t i = 0; i < len; i++) { + if ((i % sizeof(int)) == 0) { + val = pad; + } + uint8_t c = static_cast(tolower(msg[i])); + val = c + (val << 8); /* hash shift size 8 */ + if ((i % 4) == 3) { /* msg size 4, shift when 3 */ + *buf++ = val; + val = pad; + num--; + } + } + if (--num >= 0) { + *buf++ = val; + } + while (--num >= 0) { + *buf++ = pad; + } +} + +static void TeaTransform(uint32_t buf[4], uint32_t const in[]) +{ + int n = 16; /* transform total rounds 16 */ + uint32_t a = in[0]; /* transform input pos 0 */ + uint32_t b = in[1]; /* transform input pos 1 */ + uint32_t c = in[2]; /* transform input pos 2 */ + uint32_t d = in[3]; /* transform input pos 3 */ + uint32_t b0 = buf[0]; /* buf pos 0 */ + uint32_t b1 = buf[1]; /* buf pos 1 */ + uint32_t sum = 0; + + do { + sum += DELTA; + b0 += ((b1 << 4) + a) ^ (b1 + sum) ^ ((b1 >> 5) + b); /* tea transform width 4 and 5 */ + b1 += ((b0 << 4) + c) ^ (b0 + sum) ^ ((b0 >> 5) + d); /* tea transform width 4 and 5 */ + } while (--n); + + buf[0] += b0; + buf[1] += b1; +} + +static uint32_t DentryHash(const std::string &name) +{ + if (IsDotDotdot(name)) { + return 0; + } + + constexpr int inLen = 8; /* hash input buf size 8 */ + constexpr int bufLen = 4; /* hash output buf size 4 */ + uint32_t in[inLen], buf[bufLen]; + auto len = name.length(); + constexpr decltype(len) hashWidth = 16; /* hash operation width 4 */ + const char *p = name.c_str(); + + buf[0] = 0x67452301; /* hash magic 1 */ + buf[1] = 0xefcdab89; /* hash magic 2 */ + buf[2] = 0x98badcfe; /* hash magic 3 */ + buf[3] = 0x10325476; /* hash magic 4 */ + + while (true) { + Str2HashBuf(p, len, in, bufLen); + TeaTransform(buf, in); + + if (len <= hashWidth) { + break; + } + + p += hashWidth; + len -= hashWidth; + }; + uint32_t hash = buf[0]; + uint32_t hmdfsHash = hash & ~HMDFS_HASH_COL_BIT; + + return hmdfsHash; +} + +static inline uint32_t GetDentrySlots(size_t nameLen) +{ + return static_cast((nameLen + BITS_PER_BYTE - 1) >> HMDFS_SLOT_LEN_BITS); +} + +static inline off_t GetDentryGroupPos(size_t bidx) +{ + return bidx * DENTRYGROUP_SIZE + DENTRYGROUP_HEADER; +} + +static inline uint64_t GetDentryGroupCnt(uint64_t size) +{ + return (size >= DENTRYGROUP_HEADER) ? ((size - DENTRYGROUP_HEADER) / DENTRYGROUP_SIZE) : 0; +} + +static uint32_t GetOverallBucket(uint32_t level) +{ + if (level >= MAX_BUCKET_LEVEL) { + LOGD("level = %{public}d overflow", level); + return 0; + } + uint64_t buckets = (1ULL << (level + 1)) - 1; + return static_cast(buckets); +} + +static size_t GetDcacheFileSize(uint32_t level) +{ + size_t buckets = GetOverallBucket(level); + return buckets * DENTRYGROUP_SIZE * BUCKET_BLOCKS + DENTRYGROUP_HEADER; +} + +static uint32_t GetBucketaddr(uint32_t level, uint32_t buckoffset) +{ + if (level >= MAX_BUCKET_LEVEL) { + return 0; + } + + uint64_t curLevelMaxBucks = (1ULL << level); + if (buckoffset >= curLevelMaxBucks) { + return 0; + } + + return static_cast(curLevelMaxBucks) + buckoffset - 1; +} + +static uint32_t GetBucketByLevel(uint32_t level) +{ + if (level >= MAX_BUCKET_LEVEL) { + LOGD("level = %{public}d overflow", level); + return 0; + } + + uint64_t buckets = (1ULL << level); + return static_cast(buckets); +} + +static uint32_t RoomForFilename(const uint8_t bitmap[], size_t slots, uint32_t maxSlots) +{ + uint32_t bitStart = 0; + + while (1) { + uint32_t zeroStart = BitOps::FindNextZeroBit(bitmap, maxSlots, bitStart); + if (zeroStart >= maxSlots) { + return maxSlots; + } + + uint32_t zeroEnd = BitOps::FindNextBit(bitmap, maxSlots, zeroStart); + if (zeroEnd - zeroStart >= slots) { + return zeroStart; + } + + bitStart = zeroEnd + 1; + if (zeroEnd + 1 >= maxSlots) { + return maxSlots; + } + } + return 0; +} + +static void UpdateDentry(HmdfsDentryGroup &d, const MetaBase &base, uint32_t nameHash, uint32_t bitPos) +{ + HmdfsDentry *de; + const std::string name = base.name; + uint32_t slots = GetDentrySlots(name.length()); + + de = &d.nsl[bitPos]; + de->hash = nameHash; + de->namelen = name.length(); + if (memcpy_s(d.fileName[bitPos], slots * DENTRY_NAME_LEN, name.c_str(), name.length())) { + LOGE("memcpy_s failed, dstLen = %{public}d, srcLen = %{public}zu", slots * DENTRY_NAME_LEN, name.length()); + } + de->atime = base.atime; + de->mtime = base.mtime; + de->size = base.size; + de->mode = base.mode; + MetaHelper::SetPosition(de, base.position); + MetaHelper::SetFileType(de, base.fileType); + if (memcpy_s(de->recordId, CLOUD_RECORD_ID_LEN, base.cloudId.c_str(), base.cloudId.length())) { + LOGE("memcpy_s failed, dstLen = %{public}d, srcLen = %{public}zu", CLOUD_RECORD_ID_LEN, base.cloudId.length()); + } + + for (uint32_t i = 0; i < slots; i++) { + BitOps::SetBit(bitPos + i, d.bitmap); + if (i) { + (de + i)->namelen = 0; + } + } +} + int32_t CloudDiskMetaFile::HandleFileByFd(unsigned long &endBlock, uint32_t &level) { + struct stat fileStat; + int err = fstat(fd_, &fileStat); + if (err < 0) { + return EINVAL; + } + if ((endBlock > GetDentryGroupCnt(fileStat.st_size)) && + ftruncate(fd_, GetDcacheFileSize(level))) { + return ENOENT; + } return E_OK; } @@ -172,24 +384,206 @@ struct DcacheLookupCtx { std::unique_ptr page{nullptr}; }; +static void InitDcacheLookupCtx(DcacheLookupCtx *ctx, const MetaBase &base, int fd) +{ + ctx->fd = fd; + ctx->name = base.name; + ctx->bidx = 0; + ctx->page = nullptr; + ctx->hash = DentryHash(ctx->name); +} + +static std::unique_ptr FindDentryPage(uint64_t index, DcacheLookupCtx *ctx) +{ + auto dentryBlk = std::make_unique(); + + off_t pos = GetDentryGroupPos(index); + ssize_t size = FileUtils::ReadFile(ctx->fd, pos, DENTRYGROUP_SIZE, dentryBlk.get()); + if (size != DENTRYGROUP_SIZE) { + return nullptr; + } + return dentryBlk; +} + +static HmdfsDentry *FindInBlock(HmdfsDentryGroup &dentryBlk, uint32_t namehash, const std::string &name) +{ + int maxLen = 0; + uint32_t bitPos = 0; + HmdfsDentry *de = nullptr; + + while (bitPos < DENTRY_PER_GROUP) { + if (!BitOps::TestBit(bitPos, dentryBlk.bitmap)) { + bitPos++; + maxLen++; + continue; + } + de = &dentryBlk.nsl[bitPos]; + if (!de->namelen) { + bitPos++; + continue; + } + + if (de->hash == namehash && de->namelen == name.length() && + !memcmp(name.c_str(), dentryBlk.fileName[bitPos], de->namelen)) { + return de; + } + maxLen = 0; + bitPos += GetDentrySlots(de->namelen); + } + + return nullptr; +} + +static HmdfsDentry *InLevel(uint32_t level, DcacheLookupCtx *ctx) +{ + HmdfsDentry *de = nullptr; + + uint32_t nbucket = GetBucketByLevel(level); + if (!nbucket) { + return de; + } + + uint32_t bidx = GetBucketaddr(level, ctx->hash % nbucket) * BUCKET_BLOCKS; + uint32_t endBlock = bidx + BUCKET_BLOCKS; + + for (; bidx < endBlock; bidx++) { + auto dentryBlk = FindDentryPage(bidx, ctx); + if (dentryBlk == nullptr) { + break; + } + + de = FindInBlock(*dentryBlk, ctx->hash, ctx->name); + if (de != nullptr) { + ctx->page = std::move(dentryBlk); + break; + } + } + ctx->bidx = bidx; + return de; +} + +static HmdfsDentry *FindDentry(DcacheLookupCtx *ctx) +{ + for (uint32_t level = 0; level < MAX_BUCKET_LEVEL; level++) { + HmdfsDentry *de = InLevel(level, ctx); + if (de != nullptr) { + return de; + } + } + return nullptr; +} + int32_t CloudDiskMetaFile::DoRemove(const MetaBase &base) { + if (fd_ < 0) { + LOGE("bad metafile fd"); + return EINVAL; + } + + std::unique_lock lock(mtx_); + FileRangeLock fileLock(fd_, 0, 0); + DcacheLookupCtx ctx; + InitDcacheLookupCtx(&ctx, base, fd_); + HmdfsDentry *de = FindDentry(&ctx); + if (de == nullptr) { + LOGE("find dentry failed"); + return ENOENT; + } + + uint32_t bitPos = (de - ctx.page->nsl); + uint32_t slots = GetDentrySlots(de->namelen); + for (uint32_t i = 0; i < slots; i++) { + BitOps::ClearBit(bitPos + i, ctx.page->bitmap); + } + + off_t ipos = GetDentryGroupPos(ctx.bidx); + ssize_t size = FileUtils::WriteFile(fd_, ctx.page.get(), ipos, sizeof(HmdfsDentryGroup)); + if (size != sizeof(HmdfsDentryGroup)) { + LOGE("WriteFile failed!, ret = %{public}zd", size); + return EIO; + } + + --dentryCount_; return E_OK; } int32_t CloudDiskMetaFile::DoLookup(MetaBase &base) { + if (fd_ < 0) { + LOGE("bad metafile fd"); + return EINVAL; + } + + std::unique_lock lock(mtx_); + FileRangeLock fileLock(fd_, 0, 0); + struct DcacheLookupCtx ctx; + InitDcacheLookupCtx(&ctx, base, fd_); + struct HmdfsDentry *de = FindDentry(&ctx); + if (de == nullptr) { + LOGD("find dentry failed"); + return ENOENT; + } + + base.size = de->size; + base.atime = de->atime; + base.mtime = de->mtime; + base.mode = de->mode; + base.position = MetaHelper::GetPosition(de); + base.fileType = MetaHelper::GetFileType(de); + base.cloudId = std::string(reinterpret_cast(de->recordId), CLOUD_RECORD_ID_LEN); return E_OK; } int32_t CloudDiskMetaFile::DoUpdate(const MetaBase &base) { + if (fd_ < 0) { + LOGE("bad metafile fd"); + return EINVAL; + } + + std::unique_lock lock(mtx_); + FileRangeLock fileLock(fd_, 0, 0); + struct DcacheLookupCtx ctx; + InitDcacheLookupCtx(&ctx, base, fd_); + struct HmdfsDentry *de = FindDentry(&ctx); + if (de == nullptr) { + LOGD("find dentry failed"); + return ENOENT; + } + + de->atime = base.atime; + de->mtime = base.mtime; + de->size = base.size; + de->mode = base.mode; + MetaHelper::SetPosition(de, base.position); + MetaHelper::SetFileType(de, base.fileType); + if (memcpy_s(de->recordId, CLOUD_RECORD_ID_LEN, base.cloudId.c_str(), base.cloudId.length())) { + LOGE("memcpy_s failed, dstLen = %{public}d, srcLen = %{public}zu", CLOUD_RECORD_ID_LEN, base.cloudId.length()); + } + + off_t ipos = GetDentryGroupPos(ctx.bidx); + ssize_t size = FileUtils::WriteFile(fd_, ctx.page.get(), ipos, sizeof(struct HmdfsDentryGroup)); + if (size != sizeof(struct HmdfsDentryGroup)) { + LOGE("write failed, ret = %{public}zd", size); + return EIO; + } return E_OK; } int32_t CloudDiskMetaFile::DoRename(MetaBase &metaBase, const std::string &newName, std::shared_ptr newMetaFile) { + int32_t ret = DoRemove(metaBase); + if (ret != E_OK) { + LOGE("remove dentry failed, ret = %{public}d", ret); + return ret; + } + metaBase.name = newName; + ret = newMetaFile->DoCreate(metaBase); + if (ret != E_OK) { + LOGE("create dentry failed, ret = %{public}d", ret); + return ret; + } return E_OK; } -- Gitee