From 2e281f6d269773b7c5b1325df1e51137c85cb249 Mon Sep 17 00:00:00 2001 From: kazu Date: Sun, 21 Feb 2021 21:04:11 +0100 Subject: [PATCH] worked on FAT stuff and tests --- ext/sd/fat32/DirHandle.h | 146 +++++++++++++++++++++++ ext/sd/fat32/DirHelper.h | 130 +++++++++++++++++++++ ext/sd/fat32/DirIterator.h | 74 +++++++----- ext/sd/fat32/FS.h | 172 +++++++++++++++++----------- ext/sd/fat32/File.h | 132 ++++++++++++++++++--- ext/sd/fat32/File2.h | 178 ----------------------------- ext/sd/fat32/FreeClusterIterator.h | 8 +- ext/sd/fat32/Helper.h | 13 +++ ext/sd/fat32/Structs.h | 123 ++++++-------------- ext/sd/fat32/UsedSpaceChecker.h | 25 ++++ ext/sd/main.cpp | 14 +-- ext/sd/tests/TestCreate.cpp | 139 ++++++++++++++++++++-- 12 files changed, 758 insertions(+), 396 deletions(-) create mode 100644 ext/sd/fat32/DirHandle.h create mode 100644 ext/sd/fat32/DirHelper.h delete mode 100644 ext/sd/fat32/File2.h create mode 100644 ext/sd/fat32/Helper.h create mode 100644 ext/sd/fat32/UsedSpaceChecker.h diff --git a/ext/sd/fat32/DirHandle.h b/ext/sd/fat32/DirHandle.h new file mode 100644 index 0000000..be1a064 --- /dev/null +++ b/ext/sd/fat32/DirHandle.h @@ -0,0 +1,146 @@ +#include "Structs.h" +#include "Helper.h" + +namespace FAT32 { + + /** combine a DirectoryEntry with its absolute location on disk */ + class DirHandle { + + friend class File; + friend class DirIterator; + + + template friend class FS; + + /** the absolute byte location on the disk where the underlying DirEntry is stored */ + AbsPos posOnDisk; + + /** the FAT32 entry format */ + DirEntry entry; + + /** is this a valid handle? */ + bool valid; + + public: + + /** ctor for an invalid handle */ + DirHandle() : valid(false) {} + + /** ctor for a valid handle */ + DirHandle(AbsPos posOnDisk, DirEntry* entry) : posOnDisk(posOnDisk), entry(*entry), valid(true) {} + + static DirHandle invalid() {return DirHandle();} + + /** is this a valid handle? */ + bool isValid() const {return valid;} + + + /** get the entry's size */ + uint32_t getSize() const {return entry.size;} + + /** get the entry's name (filename / foldername) */ + std::string getName() const { + + char buf[16]; + + // the name + uint8_t pos = 0; + for (uint8_t i = 0; i < 8; ++i) {if (entry.name[i] != ' ') {buf[pos++] = entry.name[i];}} + + // the extension (if any, folder have none) + if (entry.ext[0] != ' ') { + buf[pos++] = '.'; + for (uint8_t i = 0; i < 3; ++i) {if ( entry.ext[i] != ' ') {buf[pos++] = entry.ext[i];}} + } + + // null terminate and return + buf[pos] = 0; + + std::string str(buf); + toLower(str); + return str; + + } + + bool isUnused() const {return entry.name[0] == 0xE5;} + bool isDirectory() const {return entry.attr.bits.directory;} + bool isEndOfDirectory() const {return entry.name[0] == 0x00;} + bool isLongFileName() const {return (entry.attr.raw & 0b1111) == 0b1111;} + + bool isDot() const {return entry.name[0] == '.' && entry.name[1] == ' ';} + bool isDotDot() const {return entry.name[0] == '.' && entry.name[1] == '.' && entry.name[2] == ' ';} + + ClusterNr getFirstCluster() const {return entry.firstClusterHi<<16 | entry.firstClusterLo<<0;} + + bool hasExtension(const char* ext) const { + const size_t len = strlen(ext); + if (len > 3) {return false;} + for (int i = 0; i < len; ++i) { + if (asciitolower(entry.ext[i]) != asciitolower(ext[i])) {return false;} + } + for (int i = len; i < 3; ++i) { + if (entry.ext[i] != ' ') {return false;} + } + return true; + } + + private: + + void setSize(uint32_t size) {entry.size = size;} + + /** this entry is an end-of-directory marker */ + void setEndOfDirectory() { + entry.name[0] = 0x00; + } + + /** this entry is a directory */ + void setDirectory() { + setSize(0); + entry.attr.bits.directory = 1; + } + + /** this entry is the '.' pointer (to self) */ + void setDot(ClusterNr myClusterNr) { + setSize(0); + entry.attr.bits.directory = 1; // not explicitly pointed out within manual, but e.g. linux needs it + memcpy(entry.name83, ". ", 8+3); + setFirstCluster(myClusterNr); + } + + /** this entry is the '.' pointer (to parent) */ + void setDotDot(ClusterNr parentClusterNr) { + setSize(0); + entry.attr.bits.directory = 1; // not explicitly pointed out within manual, but e.g. linux needs it + memcpy(entry.name83, ".. ", 8+3); + setFirstCluster(parentClusterNr); + } + + void setFirstCluster(ClusterNr nr) {entry.firstClusterHi = (uint16_t)(nr << 16); entry.firstClusterLo = (uint16_t)(nr << 0);} + + void setName(const std::string& str) { + + // "zero" fill name and extension (0x20) + memset(entry.name83, ' ', 11); + + auto pos = str.find('.'); + + // name + for (size_t i = 0; i < min(8, min(pos, str.length())); ++i) { + if (i >= 8) {break;} + entry.name[i] = str[i]; + } + + // extension (if any, folders have none) + if (pos != std::string::npos) { + uint32_t extLen = min(3, str.length() - pos - 1); + for (size_t i = 0; i < extLen; ++i) { + if (i >= 3) {break;} + entry.ext[i] = str[i+pos+1]; + } + } + + } + + }; + +} diff --git a/ext/sd/fat32/DirHelper.h b/ext/sd/fat32/DirHelper.h new file mode 100644 index 0000000..377dfa1 --- /dev/null +++ b/ext/sd/fat32/DirHelper.h @@ -0,0 +1,130 @@ +#pragma once + +class DirHelper { + + static constexpr const char* NAME = "FAT32_DirH"; + +public: + + enum class CreateType { + NONE, + DIR, + FILE + }; + + /** + * get (or allocate, if needed) a new DirEntry with the given name absolute name + * if the entry is allocated, it is UNTYPED + * name: e.g. /path1/path2/path3 or /path1/path2/file.txt + */ + static DirHandle getOrCreate(FS& fs, ClusterNr dirStartCluster, const std::string& absName, CreateType ct) { + + // determine the local name + Split s = split(absName); + + { // search for a matching existing entry + DirIterator di(fs, dirStartCluster); + while(true) { + + DirHandle h = di.nextUsable(); + if (!h.isValid()) {break;} + + // found a matching entry + const std::string n = h.getName(); + if (n == s.first) { + + // are we done yet? + if (s.hasMore()) { + const ClusterNr subDirStartCluster = h.getFirstCluster(); + return getOrCreate(fs, subDirStartCluster, s.next, ct); + } else { + return h; + } + + } + } + } + + // no matching entry found + if (ct == CreateType::NONE) {return DirHandle::invalid();} + + { // allocate a new DirEntry within the current directory + + DirIterator di(fs, dirStartCluster); + DirHandle h = di.nextFree(); + h.setName(s.first); + h.setSize(0); + h.setFirstCluster(0); + + // if this is a sub-directory, allocate a new cluster for it + if (s.hasMore() || ct == CreateType::DIR) { + + // allocate a cluster where the directory contents are to be stored + ClusterNr freeCluster = fs.allocFreeCluster(0); + h.setFirstCluster(freeCluster); + h.setDirectory(); + fs.write(h); + + // ensure the new cluster is initialized with all zeros (this is also equal to an immediate END_OF_DIRECTORY marker in the beginning + fs.zeroOutCluster(freeCluster); + + DirIterator di2(fs, freeCluster); + + // create the "." entry (pointer to current directory) + DirHandle h2 = di2.next(); + h2.setDot(freeCluster); + fs.write(h2); + + // create the ".." entry (pointer to parent directory) + DirHandle h3 = di2.next(); + h3.setDotDot(dirStartCluster); + fs.write(h3); + + // more path to process? + if (s.hasMore()) { + return getOrCreate(fs, freeCluster, s.next, ct); + } else { + return h; + } + + } else { + + fs.write(h); + return h; + + } + + } + + } + +private: + + struct Split { + + std::string first; + std::string next; + + Split(const std::string& first, const std::string& next) : first(first), next(next) {} + + bool hasMore() const {return !next.empty();} + + }; + + /** split "/path1/path2/path3/file.txt" into "path1" and "path2/path3/file.txt" */ + static Split split(std::string absPath) { + + if (absPath[0] == '/') { + absPath = absPath.substr(1); + } + + std::size_t pos = absPath.find('/'); + if (pos == std::string::npos) { + return Split(absPath, ""); + } else { + return Split(absPath.substr(0, pos), absPath.substr(pos)); + } + + } + +}; diff --git a/ext/sd/fat32/DirIterator.h b/ext/sd/fat32/DirIterator.h index d8f90c7..d83d0cf 100644 --- a/ext/sd/fat32/DirIterator.h +++ b/ext/sd/fat32/DirIterator.h @@ -15,63 +15,73 @@ public: Log::addInfo(NAME, "init @ Cluster %d", curCluster); + // ".." folders point to cluster 0 if they point to the root folder -> adjust here + if (clusterNr == 0) { + clusterNr = fs.desc.rootDirFirstCluster; + } + // read the first sector in the first cluster read(curCluster, 0); } + DirIterator(FS& fs, DirHandle dh) : DirIterator(fs, dh.getFirstCluster()) { + + } + + /** get the next usable entry within the current directory */ - DirEntryAt nextUsable() { + DirHandle nextUsable() { while(true) { - DirEntryAt dea = next(false); + DirHandle h = next(false); // check it - if (!dea.isValid()) {return DirEntryAt::invalid();} - if (dea.isLongFileName()) {continue;} - if (dea.isUnused()) {continue;} - if (dea.isEndOfDirectory()) {return DirEntryAt::invalid();} + if (!h.isValid()) {return DirHandle::invalid();} + if (h.isLongFileName()) {continue;} + if (h.isUnused()) {continue;} + if (h.isEndOfDirectory()) {return DirHandle::invalid();} // usable! - return dea; + return h; } } /** get (or create) a free entry within the current directory */ - DirEntryAt nextFree() { + DirHandle nextFree() { while(true) { - DirEntryAt dea = next(true); + DirHandle h = next(true); // check it - if (!dea.isValid()) {return DirEntryAt::invalid();} + if (!h.isValid()) {return DirHandle::invalid();} - if (dea.isUnused()) { + if (h.isUnused()) { // switch from "unused" to something usable - //dea.setName("NEW.ONE"); - //dea.entry.attr.raw = 0; + //h.setName("NEW.ONE"); + //h.entry.attr.raw = 0; //return dea; } - if (dea.isEndOfDirectory()) { + if (h.isEndOfDirectory()) { // add a new EndOfDirectory entry afterwards - DirEntryAt dea2 = next(true); - dea2.setEndOfDirectory(); - fs.write(dea2); + DirHandle h2 = next(true); + h2.setEndOfDirectory(); + fs.write(h2); // switch from "end of directory" to something usable - dea.setName("NEW.ONE"); - dea.entry.attr.raw = 0; - return dea; + h.setName("NEW.ONE"); + h.entry.attr.raw = 0; + return h; } @@ -80,15 +90,19 @@ public: } -private: - - DirEntryAt cur() { - AbsPos pos = fs.clusterToAbsPos(curCluster) + (curSectorInCluster * fs.desc.bytesPerSector) + (curEntryInSector * sizeof(DirEntry)); - DirEntry* dirEntry = reinterpret_cast(buf + (curEntryInSector * sizeof(DirEntry))); - return DirEntryAt(pos, dirEntry); + DirHandle next() { + return next(false); } - DirEntryAt next(bool allocIfNeeded) { +private: + + DirHandle cur() { + AbsPos pos = fs.clusterToAbsPos(curCluster) + (curSectorInCluster * fs.desc.bytesPerSector) + (curEntryInSector * sizeof(DirEntry)); + DirEntry* dirEntry = reinterpret_cast(buf + (curEntryInSector * sizeof(DirEntry))); + return DirHandle(pos, dirEntry); + } + + DirHandle next(bool allocIfNeeded) { while(true) { @@ -110,7 +124,7 @@ private: // do not alloc new entries? -> done here if (!allocIfNeeded) { - return DirEntryAt::invalid(); + return DirHandle::invalid(); } else { curCluster = fs.allocFreeCluster(curCluster); } @@ -129,10 +143,10 @@ private: } // the current entry - DirEntryAt dea = cur(); + DirHandle h = cur(); ++curEntryInSector; - return dea; + return h; } diff --git a/ext/sd/fat32/FS.h b/ext/sd/fat32/FS.h index 9684884..f431e83 100644 --- a/ext/sd/fat32/FS.h +++ b/ext/sd/fat32/FS.h @@ -5,6 +5,7 @@ #include #include "Structs.h" +#include "DirHandle.h" // https://www.pjrc.com/tech/8051/ide/fat32.html namespace FAT32 { @@ -23,41 +24,68 @@ namespace FAT32 { #include "FreeClusterIterator.h" FreeClusterIterator fci; + #include "UsedSpaceChecker.h" + public: #include "File.h" - #include "File2.h" #include "DirIterator.h" - - + #include "DirHelper.h" /** ctor with the absolute offset addr (in bytes) */ FS(BlockDev& dev, AbsOffset offset) : dev(dev), offset(offset), fci(*this) { init(); } - void setup(uint32_t totalSize) { + /** + * setup a new filesystem with the given parameters + * killFAT = zero both file allocation tables (reset all cluster numbers) + */ + void setup(uint32_t totalSize, const bool killFAT) { uint8_t buf[512] = {0}; buf[0x1FE] = 0x55; buf[0x1FF] = 0xAA; - FSHeader* header = (FSHeader*) buf; - header->bytesPerSector = 512; - header->numberOfFATs = 2; - header->numReservedSectors = 32; - header->rootDirFirstCluster = 2; - header->mediaDescriptor = 0xF8; // hard disk - header->sectorsInPartition = totalSize / 512; - header->sectorsPerCluster = 8; - header->sectorsPerFAT = 64; // 8 MB + const uint32_t sectorsPerCluster = 8; + const uint32_t sizePerCluster = 512 * sectorsPerCluster; + const uint32_t sectorsPerFAT = totalSize / sizePerCluster * sizeof(ClusterNr) / 512; + FSHeader* header = (FSHeader*) buf; + header->bytesPerSector = 512; // always + header->numberOfFATs = 2; // always + header->numReservedSectors = 32; // always + header->rootDirFirstCluster = 2; // always + header->mediaDescriptor = 0xF8; // hard disk + header->sectorsInPartition = totalSize / 512; + header->sectorsPerCluster = sectorsPerCluster; // 4096 byte clusters + header->sectorsPerFAT = sectorsPerFAT; + + // write the header to the underlying device dev.write(offset, 512, (const uint8_t*)header); + // now update the internal state based on the newly created FS header... init(); - // mark the root-dir cluster as used - setNextCluster(2, END_OF_CLUSTERS); + if (killFAT) { + + // zero-out FAT1 + uint8_t zeros[512] = {0}; + AbsPos pos = tmp.startOfFAT1; + for (uint32_t i = 0; i < header->sectorsPerFAT; ++i) { + dev.write(pos + i * 512, 512, zeros); + } + + // ...and mark the root-dir cluster as used + setNextCluster(2, END_OF_CLUSTERS); + + // set the first entry within the root-dir to denote the end-of-the-root-dir (no entries at all) + DirIterator di = getRoot(); + DirHandle dh = di.next(); + dh.setEndOfDirectory(); + write(dh); + + } } @@ -66,34 +94,64 @@ namespace FAT32 { return valid; } + /** get the total size of the filesystem (including FATs etc.) */ + uint32_t getSize() const { + return desc.sectorsPerFAT * desc.bytesPerSector / sizeof(ClusterNr) * tmp.bytesPerCluster; + } + + /** get the size that is actually usable for data (total size - required overhead) */ + uint32_t getSizeForData() const { + return getSize() - (desc.numReservedSectors * desc.bytesPerSector) - (desc.numberOfFATs * desc.sectorsPerFAT * desc.bytesPerSector); + } + + /** get the number of used data bytes */ + uint32_t getSizeUsed() const { + return UsedSpaceChecker::getNumUsedClusters(*this) * tmp.bytesPerCluster; + } + /** get an iterator for the root directory */ DirIterator getRoot() { - return DirIterator(*this, desc.rootDirFirstCluster); + return DirIterator(*this, 2); } - /** open the given file for reading*/ - File open(const DirEntryAt& dea) { - return File(*this, dea.getSize(), dea.getFirstCluster()); + /** open the given file */ + File open(const DirHandle& h) { + return File(*this, h); } - /** open the given file for reading */ - File2 open2(const DirEntryAt& dea) { - return File2(*this, dea); + /** create the given absolute folder */ + DirHandle mkdirs(const char* absName) { + DirHandle h = DirHelper::getOrCreate(*this, 2, absName, DirHelper::CreateType::DIR); + return h; + } + + /** get the entry with the given name */ + DirHandle getHandle(const char* absName) { + std::string tmp(absName); + if (tmp == "/" || tmp == "") { + DirEntry dummy; + DirHandle dh(0, &dummy); + dh.setFirstCluster(desc.rootDirFirstCluster); + return dh; + } else { + DirHandle h = DirHelper::getOrCreate(*this, 2, absName, DirHelper::CreateType::NONE); + return h; + } } /** get or create a file with the given name */ - File2 getOrCreateFile(const char* name) { + File getOrCreateFile(const char* absName) { - DirEntryAt dea = getDirEntry(name, true); + DirHandle h = DirHelper::getOrCreate(*this, 2, absName, DirHelper::CreateType::FILE); // new file -> allocate the first cluster - if (dea.getFirstCluster() == 0) { + if (h.getFirstCluster() == 0) { ClusterNr firstCluster = allocFreeCluster(0); - dea.setFirstCluster(firstCluster); - write(dea); + h.setFirstCluster(firstCluster); + write(h); } - return File2(*this, dea); + return File(*this, h); } @@ -117,49 +175,20 @@ namespace FAT32 { valid = (desc.bytesPerSector == 512) && (desc.numberOfFATs == 2) && (getU16(&buf[0x1FE]) == 0xAA55); tmp.bytesPerCluster = desc.sectorsPerCluster * desc.bytesPerSector; - tmp.startOfFAT = offset + (desc.numReservedSectors * desc.bytesPerSector); + tmp.startOfFAT1 = offset + (desc.numReservedSectors * desc.bytesPerSector); + tmp.startOfFAT2 = tmp.startOfFAT1 + desc.sectorsPerFAT * desc.bytesPerSector; tmp.startOfFirstDataCluster = offset + (desc.numReservedSectors * desc.bytesPerSector) + (desc.numberOfFATs * desc.sectorsPerFAT * desc.bytesPerSector); tmp.startOfFirstRootDirCluster = clusterToAbsPos(desc.rootDirFirstCluster); + tmp.entriesPerFAT = desc.sectorsPerFAT * desc.bytesPerSector / sizeof(ClusterNr); tmp.dirEntriesPerSector = desc.bytesPerSector / sizeof(DirEntry); Log::addInfo(NAME, "Bytes/Sector: %d, Sector/Cluster: %d, FATs: %d, RootDir: %d", desc.bytesPerSector, desc.sectorsPerCluster, desc.numberOfFATs, desc.rootDirFirstCluster); } - /** get (or create, if needed) a DirEntry with the given name */ - DirEntryAt getDirEntry(const char* name, bool createIfNeeded) { - - // TODO: support for sub folders) - - // start at the root folder - ClusterNr dirStartCluster = 2; - - { // search for a matching existing entry - DirIterator di(*this, dirStartCluster); - while(true) { - DirEntryAt dea = di.nextUsable(); - if (!dea.isValid()) {break;} - if (dea.getName() == name) {return dea;} - } - } - - // no matching entry found - if (!createIfNeeded) {return DirEntryAt::invalid();} - - { // allocate a new DirEntry for the file - DirIterator di(*this, dirStartCluster); - DirEntryAt dea = di.nextFree(); - dea.setName(name); - dea.setSize(0); - dea.setFirstCluster(0); - return dea; - } - - } - /** determine the ClusterNr following the given ClusterNr */ - ClusterNr getNextCluster(const ClusterNr clusterNr) { - const AbsPos pos = tmp.startOfFAT + (clusterNr * sizeof(ClusterNr)); + ClusterNr getNextCluster(const ClusterNr clusterNr) const { + const AbsPos pos = tmp.startOfFAT1 + (clusterNr * sizeof(ClusterNr)); ClusterNr next = 0; dev.read(pos, sizeof(ClusterNr), reinterpret_cast(&next)); Log::addInfo(NAME, "getNextCluster(%d) -> %d", clusterNr, next); @@ -168,8 +197,8 @@ namespace FAT32 { /** set the ClusterNr following clusterNr */ void setNextCluster(const ClusterNr clusterNr, const ClusterNr next) { - const AbsPos pos = tmp.startOfFAT + (clusterNr * sizeof(ClusterNr)); - dev.write(pos, sizeof(ClusterNr), reinterpret_cast(&next)); + const AbsPos pos = tmp.startOfFAT1 + (clusterNr * sizeof(ClusterNr)); + dev.write(pos, sizeof(ClusterNr), reinterpret_cast(&next.val)); Log::addInfo(NAME, "setNextCluster(%d) -> %d", clusterNr, next); } @@ -188,9 +217,18 @@ namespace FAT32 { return newCluster; } - /** write the given DirEntry back to disk */ - void write(const DirEntryAt& dea) { - dev.write(dea.posOnDisk, sizeof(DirEntry), (const uint8_t*)&dea.entry); + /** write all zeros to the given cluster */ + void zeroOutCluster(ClusterNr nr) { + uint8_t zeros[512] = {0}; + AbsPos pos = clusterToAbsPos(nr); + for (uint8_t i = 0; i < desc.sectorsPerCluster; ++i) { + dev.write(pos + i * 512, 512, zeros); + } + } + + /** write the given DirHandle back to disk */ + void write(const DirHandle& h) { + dev.write(h.posOnDisk, sizeof(DirEntry), (const uint8_t*)&h.entry); } }; diff --git a/ext/sd/fat32/File.h b/ext/sd/fat32/File.h index 223793b..1949eec 100644 --- a/ext/sd/fat32/File.h +++ b/ext/sd/fat32/File.h @@ -4,25 +4,36 @@ class File { static constexpr const uint32_t F_EOF = 0xFFFFFFFF; FS& fs; - uint32_t totalSize; + DirHandle h; - uint32_t curAbsPos = 0; // position withithin the whole file - ClusterNr curCluster = 0; // cluster we are currently reading + uint32_t curAbsPos = 0; // absolute position withithin the file uint32_t posInCluster = 0; // position within the current cluster + uint32_t iCurCluster = 0; // current cluster index (within list of clusters) - - + std::vector clusters; // all clusters the file uses public: - File(FS& fs, uint32_t size, ClusterNr firstCluster) : fs(fs), totalSize(size), curCluster(firstCluster) { - Log::addInfo(NAME, "init @ cluster %d", firstCluster); + File(FS& fs, const DirHandle& h) : fs(fs), h(h) { + Log::addInfo(NAME, "init @ cluster %d", getFirstCluster()); + getAllClusters(h.getFirstCluster()); } - /** the file's size */ - uint32_t getSize() const {return totalSize;} + /** the file's USED size */ + uint32_t getSize() const {return h.getSize();} - uint32_t read(uint32_t size, uint8_t* dst, std::function callback) { + /** the file's ALLOCATED size */ + uint32_t getAllocatedSize() const {return clusters.size() * fs.tmp.bytesPerCluster;} + + /** the file's first cluster */ + ClusterNr getFirstCluster() const {return h.getFirstCluster();} + + /** get the file's name */ + std::string getName() const {return h.getName();} + + + /** read x bytes from the file */ + uint32_t read(uint32_t size, uint8_t* dst) { Log::addInfo(NAME, "read %d bytes", size); @@ -37,27 +48,75 @@ public: remaining -= read; dst += read; totalRead += read; - callback(totalRead*100/size); } return totalRead; } + /* write the given data into the file */ + uint32_t write(uint32_t size, const uint8_t* src) { + + Log::addInfo(NAME, "write %d bytes", size); + + uint32_t remaining = size; + uint32_t totalWritten = 0; + + while(remaining) { + const uint32_t written = _write(remaining, src); + remaining -= written; + src += written; + totalWritten += written; + //callback(totalWritten*100/size); + } + + // update the file header (filesize might have changed) + fs.write(h); + + return totalWritten; + + } + private: + void setSize(uint32_t size) {h.setSize(size);} + + /** fetch the list of all clusters the file is using */ + void getAllClusters(ClusterNr startCluster) { + + // initial cluster + clusters.push_back(startCluster); + + // all following clusters + while(true) { + startCluster = fs.getNextCluster(startCluster); + if (!startCluster.isEndOfClusters()) { + clusters.push_back(startCluster); + } else { + break; + } + } + + // debug + //char buf[1024]; + //char* dst = buf; dst += sprintf(dst, "clusters: "); + //for (ClusterNr nr : clusters) {dst += sprintf(dst, "%d ", nr);} + //Log::addInfo(NAME, buf); + + } + int32_t _read(uint32_t readSize, uint8_t* dst) { // EOF reached? - if (curAbsPos >= totalSize) { + if (curAbsPos >= getSize()) { return F_EOF; } - // end of current cluster reached? -> determine the next one + // end of current cluster reached? -> switch to the next one if (posInCluster == fs.tmp.bytesPerCluster) { - curCluster = fs.getNextCluster(curCluster); + ++iCurCluster; posInCluster = 0; - Log::addInfo(NAME, "next cluster %d", curCluster); + Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]); } // how many bytes are left in the current cluster? @@ -66,7 +125,7 @@ private: // determine how many bytes to read const uint32_t toRead = std::min(remainingInCluster, readSize); - const uint32_t offset = fs.clusterToAbsPos(curCluster) + posInCluster; + const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster; const uint32_t read = fs.dev.read(offset, toRead, dst); posInCluster += read; curAbsPos += read; @@ -75,4 +134,45 @@ private: } + int32_t _write(uint32_t writeSize, const uint8_t* src) { + + // do we need to append more free sectors to the end of the file? + if (curAbsPos >= getAllocatedSize()) { + if (clusters.empty()) { + const ClusterNr newCluster = fs.allocFreeCluster(0); + clusters.push_back(newCluster); + Log::addInfo(NAME, "allocFirstCluster: %d", newCluster); + } else { + const ClusterNr prevCluster = clusters.back(); + const ClusterNr newCluster = fs.allocFreeCluster(prevCluster); + clusters.push_back(newCluster); + Log::addInfo(NAME, "allocNextCluster(%d): %d", prevCluster, newCluster); + } + } + + // end of current cluster reached? -> switch to the next one + if (posInCluster == fs.tmp.bytesPerCluster) { + ++iCurCluster; + posInCluster = 0; + Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]); + } + + // how many bytes are left in the current cluster? + const uint32_t remainingInCluster = fs.tmp.bytesPerCluster - posInCluster; + + // determine how many bytes to write + const uint32_t toWrite = std::min(remainingInCluster, writeSize); + + const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster; + const uint32_t written = fs.dev.write(offset, toWrite, src); + posInCluster += written; + curAbsPos += written; + + // adjust the number of bytes used + if (this->getSize() < curAbsPos) {this->setSize(curAbsPos);} + + return written; + + } + }; diff --git a/ext/sd/fat32/File2.h b/ext/sd/fat32/File2.h deleted file mode 100644 index b5b69f4..0000000 --- a/ext/sd/fat32/File2.h +++ /dev/null @@ -1,178 +0,0 @@ -class File2 { - - static constexpr const char* NAME = "FAT32_File2"; - static constexpr const uint32_t F_EOF = 0xFFFFFFFF; - - FS& fs; - DirEntryAt dea; - - uint32_t curAbsPos = 0; // absolute position withithin the file - uint32_t posInCluster = 0; // position within the current cluster - uint32_t iCurCluster = 0; // current cluster index (within list of clusters) - - std::vector clusters; // all clusters the file uses - -public: - - File2(FS& fs, const DirEntryAt& dea) : fs(fs), dea(dea) { - Log::addInfo(NAME, "init @ cluster %d", getFirstCluster()); - getAllClusters(dea.getFirstCluster()); - } - - /** the file's USED size */ - uint32_t getSize() const {return dea.getSize();} - - /** the file's ALLOCATED size */ - uint32_t getAllocatedSize() const {return clusters.size() * fs.tmp.bytesPerCluster;} - - /** the file's first cluster */ - ClusterNr getFirstCluster() const {return dea.getFirstCluster();} - - /** get the file's name */ - std::string getName() const {return dea.getName();} - - - /** read x bytes from the file */ - uint32_t read(uint32_t size, uint8_t* dst) { - - Log::addInfo(NAME, "read %d bytes", size); - - uint32_t remaining = size; - uint32_t totalRead = 0; - - while(remaining) { - const uint32_t read = _read(remaining, dst); - if (read == F_EOF) { - Log::addInfo(NAME, "EOF"); break; - } - remaining -= read; - dst += read; - totalRead += read; - } - - return totalRead; - - } - - /* write the given data into the file */ - uint32_t write(uint32_t size, const uint8_t* src) { - - Log::addInfo(NAME, "write %d bytes", size); - - uint32_t remaining = size; - uint32_t totalWritten = 0; - - while(remaining) { - const uint32_t written = _write(remaining, src); - remaining -= written; - src += written; - totalWritten += written; - //callback(totalWritten*100/size); - } - - // update the file header (filesize might have changed) - fs.write(dea); - - return totalWritten; - - } - -private: - - void setSize(uint32_t size) {dea.setSize(size);} - - /** fetch the list of all clusters the file is using */ - void getAllClusters(ClusterNr startCluster) { - - // initial cluster - clusters.push_back(startCluster); - - // all following clusters - while(true) { - startCluster = fs.getNextCluster(startCluster); - if (!startCluster.isEndOfClusters()) { - clusters.push_back(startCluster); - } else { - break; - } - } - - // debug - //char buf[1024]; - //char* dst = buf; dst += sprintf(dst, "clusters: "); - //for (ClusterNr nr : clusters) {dst += sprintf(dst, "%d ", nr);} - //Log::addInfo(NAME, buf); - - } - - int32_t _read(uint32_t readSize, uint8_t* dst) { - - // EOF reached? - if (curAbsPos >= getSize()) { - return F_EOF; - } - - // end of current cluster reached? -> switch to the next one - if (posInCluster == fs.tmp.bytesPerCluster) { - ++iCurCluster; - posInCluster = 0; - Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]); - } - - // how many bytes are left in the current cluster? - const uint32_t remainingInCluster = fs.tmp.bytesPerCluster - posInCluster; - - // determine how many bytes to read - const uint32_t toRead = std::min(remainingInCluster, readSize); - - const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster; - const uint32_t read = fs.dev.read(offset, toRead, dst); - posInCluster += read; - curAbsPos += read; - - return read; - - } - - int32_t _write(uint32_t writeSize, const uint8_t* src) { - - // do we need to append more free sectors to the end of the file? - if (curAbsPos >= getAllocatedSize()) { - if (clusters.empty()) { - const ClusterNr newCluster = fs.allocFreeCluster(0); - clusters.push_back(newCluster); - Log::addInfo(NAME, "allocFirstCluster: %d", newCluster); - } else { - const ClusterNr prevCluster = clusters.back(); - const ClusterNr newCluster = fs.allocFreeCluster(prevCluster); - clusters.push_back(newCluster); - Log::addInfo(NAME, "allocNextCluster(%d): %d", prevCluster, newCluster); - } - } - - // end of current cluster reached? -> switch to the next one - if (posInCluster == fs.tmp.bytesPerCluster) { - ++iCurCluster; - posInCluster = 0; - Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]); - } - - // how many bytes are left in the current cluster? - const uint32_t remainingInCluster = fs.tmp.bytesPerCluster - posInCluster; - - // determine how many bytes to write - const uint32_t toWrite = std::min(remainingInCluster, writeSize); - - const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster; - const uint32_t written = fs.dev.write(offset, toWrite, src); - posInCluster += written; - curAbsPos += written; - - // adjust the number of bytes used - if (this->getSize() < curAbsPos) {this->setSize(curAbsPos);} - - return written; - - } - -}; diff --git a/ext/sd/fat32/FreeClusterIterator.h b/ext/sd/fat32/FreeClusterIterator.h index 9ef3e30..d8c80e1 100644 --- a/ext/sd/fat32/FreeClusterIterator.h +++ b/ext/sd/fat32/FreeClusterIterator.h @@ -1,6 +1,10 @@ #pragma once -/** helper class to iterate all free clusters of the Filesystem */ +/** + * helper class to iterate all free clusters of the Filesystem. + * starts at cluster number 2 and then iterates over all subsequent clusters + * stopping at the free ones + */ class FreeClusterIterator { FS& fs; @@ -15,6 +19,8 @@ public: /** get the next free cluster that is available */ ClusterNr next() { + // TODO: end of filesystem reached + while(true) { const ClusterNr tmp = fs.getNextCluster(cur); if (tmp.isFree()) {return cur;} diff --git a/ext/sd/fat32/Helper.h b/ext/sd/fat32/Helper.h new file mode 100644 index 0000000..0543ea3 --- /dev/null +++ b/ext/sd/fat32/Helper.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +static char asciitolower(char in) { + if (in <= 'Z' && in >= 'A') {return in - ('Z' - 'z');} + return in; +} + +static void toLower(std::string& str) { + std::transform(str.begin(), str.end(), str.begin(), asciitolower); +} diff --git a/ext/sd/fat32/Structs.h b/ext/sd/fat32/Structs.h index 10e85a0..f5272a5 100644 --- a/ext/sd/fat32/Structs.h +++ b/ext/sd/fat32/Structs.h @@ -6,12 +6,12 @@ namespace FAT32 { -static inline uint16_t getU8(const uint8_t* src) {return src[0];} -static inline uint16_t getU16(const uint8_t* src) {return src[0]<<0 | src[1]<<8;} -static inline uint32_t getU32(const uint8_t* src) {return src[0]<<0 | src[1]<<8 | src[2]<<16 | src[3]<<24;} -static inline size_t min(size_t a, size_t b) {return (a= 8) {break;} - entry.name[i] = str[i]; - } - for (size_t i = 0; i < min(pos, str.length()); ++i) { - if (i >= 3) {break;} - entry.ext[i] = str[i+pos+1]; - } - - } + + + /** only the important parts of above header (to save space) */ + struct FSDesc { + uint16_t bytesPerSector; + uint8_t sectorsPerCluster; + uint16_t numReservedSectors; + uint8_t numberOfFATs; + uint32_t sectorsPerFAT; + ClusterNr rootDirFirstCluster; }; + /** additional pre-computed values, to reduce calculations */ + struct Precomputed { + uint32_t bytesPerCluster; + AbsPos startOfFAT1; // absolute byte offset where the FAT1 begins + AbsPos startOfFAT2; // absolute byte offset where the FAT2 begins + AbsPos startOfFirstDataCluster; // absolute byte offset where the first cluster is + AbsPos startOfFirstRootDirCluster; // absolute byte offset where the first root dir cluster is + uint32_t entriesPerFAT; // number of entries (cluster pointers) within each FAT + uint32_t dirEntriesPerSector; // number of directory entries that fit into a sector + }; } diff --git a/ext/sd/fat32/UsedSpaceChecker.h b/ext/sd/fat32/UsedSpaceChecker.h new file mode 100644 index 0000000..395a023 --- /dev/null +++ b/ext/sd/fat32/UsedSpaceChecker.h @@ -0,0 +1,25 @@ +#pragma once + +/** + * helper class to determine all clusters whice are used. + * starts at cluster number 2 and then iterates over all subsequent clusters + */ +class UsedSpaceChecker { + +public: + + /** get the number of used clusters */ + static uint32_t getNumUsedClusters(const FS& fs) { + + uint32_t numUsed = 0; + + for (ClusterNr nr = 2; nr < fs.tmp.entriesPerFAT; ++nr) { + ClusterNr next = fs.getNextCluster(nr); + if (!next.isFree()) {++numUsed;} + } + + return numUsed; + + } + +}; diff --git a/ext/sd/main.cpp b/ext/sd/main.cpp index 7849480..06487b5 100644 --- a/ext/sd/main.cpp +++ b/ext/sd/main.cpp @@ -112,11 +112,11 @@ int main(int argc, char** argv) { FAT32FS::DirIterator dir = fat.getRoot(); while(false) { - FAT32::DirEntryAt dea = dir.nextUsable(); - if (!dea.isValid()) {break;} + FAT32::DirHandle h = dir.nextUsable(); + if (!h.isValid()) {break;} if (1==0) { - FAT32FS::File2 f = fat.open2(dea); + FAT32FS::File f = fat.open(h); if (1==0) { uint8_t* bufff = (uint8_t*) malloc(1024*1024); @@ -137,11 +137,11 @@ int main(int argc, char** argv) { - //FAT32::DirEntryAt root = fat.getRoot().cur(); - FAT32FS::File2 file = fat.getOrCreateFile("tmp1.txt"); +// //FAT32::DirEntryAt root = fat.getRoot().cur(); +// FAT32FS::File file = fat.getOrCreateFile("tmp1.txt"); - uint8_t src[128]; - file.write(128, src); +// uint8_t src[128]; +// file.write(128, src); // diff /tmp/ram/TETRIS.GB /apps/workspace/gbemu/tests/tetris.gb // diff /tmp/ram/KIRBY1.GB /apps/workspace/gbemu/tests/Kirby\'s\ Dream\ Land\ \(USA\,\ Europe\).gb diff --git a/ext/sd/tests/TestCreate.cpp b/ext/sd/tests/TestCreate.cpp index 4a552ec..7091c3b 100644 --- a/ext/sd/tests/TestCreate.cpp +++ b/ext/sd/tests/TestCreate.cpp @@ -11,7 +11,7 @@ -TEST(TestCreat, structure) { +TEST(TestCreate, structure) { FAT32::FSHeader header; @@ -35,7 +35,7 @@ TEST (TestCreat, writeRead) { TestDevice dev(size); BlockDev bDev(dev); FS fs(bDev, 0); - fs.setup(size); + fs.setup(size, true); for (int i = 0; i < 64; ++i) { @@ -45,7 +45,7 @@ TEST (TestCreat, writeRead) { const size_t sizeA = (size/4096+1) * 4096; std::cout << name << " - " << size << std::endl; - FAT32::FS::File2 f = fs.getOrCreateFile(name); + FAT32::FS::File f = fs.getOrCreateFile(name); uint8_t* data = (uint8_t*)malloc(128 + i * 512); for (uint32_t j = 0; j < size; ++j) {data[j] = j;} uint32_t written = f.write(size, data); @@ -73,10 +73,10 @@ TEST (TestCreat, writeRead) { const size_t size = 128 + i * 512; const size_t sizeA = (size/4096+1) * 4096; - FAT32::DirEntryAt dea = di.nextUsable(); - FS::File2 f = fs.open2(dea); + FAT32::DirHandle h = di.nextUsable(); + FS::File f = fs.open(h); - ASSERT_EQ(name, dea.getName()); + ASSERT_EQ(name, h.getName()); ASSERT_EQ(name, f.getName()); ASSERT_EQ(size, f.getSize()); ASSERT_EQ(sizeA, f.getAllocatedSize()); @@ -96,7 +96,46 @@ TEST (TestCreat, writeRead) { } -TEST (TestCreate, getOrCreateFile) { +TEST (TestCreat, init) { + + using BlockDev = AccessHelper; + using FS = FAT32::FS; + + size_t size = 32*1024*1024; + TestDevice dev(size); + + // write non-zero data into the device's memory + for (uint32_t i = 0; i < size; ++i) {dev.buf[i] = 0x55;} + + BlockDev bDev(dev); + + FS fs(bDev, 0); + + // filesystem must not be considered valid, header contains only zeros + ASSERT_FALSE(fs.isValid()); + + // initialize the filesystem + fs.setup(size, true); + + // must be considered valid now + ASSERT_TRUE(fs.isValid()); + + // size indication must match 32 MiB + ASSERT_EQ(size, fs.getSize()); + + // data size indication (32 MiB - FATs and reserved sectors) + ASSERT_EQ(size - 81920, fs.getSizeForData()); + + // currently, one cluster (4096 bytes) is used (for the root dir) + ASSERT_EQ(4096, fs.getSizeUsed()); + + // there MUST NOT be any valid entry in the root dir, as this is a new FS! + FS::DirIterator di = fs.getRoot(); + ASSERT_FALSE(di.nextUsable().isValid()); + +} + +TEST (TestCreat, getOrCreateFile) { using BlockDev = AccessHelper; using FS = FAT32::FS; @@ -104,10 +143,22 @@ TEST (TestCreate, getOrCreateFile) { size_t size = 32*1024*1024; TestDevice dev(size); BlockDev bDev(dev); - FS fs(bDev, 0); - fs.setup(size); - FS::File2 f1 = fs.getOrCreateFile("test.txt"); + FS fs(bDev, 0); + + // filesystem must not be considered valid, header contains only zeros + ASSERT_FALSE(fs.isValid()); + + // initialize the filesystem + fs.setup(size, true); + + // must be considered valid now + ASSERT_TRUE(fs.isValid()); + + // size indication must match 32 MB + ASSERT_EQ(size, fs.getSize()); + + FS::File f1 = fs.getOrCreateFile("test.txt"); ASSERT_EQ(0, f1.getSize()); ASSERT_EQ(4096, f1.getAllocatedSize()); ASSERT_EQ("test.txt", f1.getName()); @@ -118,10 +169,76 @@ TEST (TestCreate, getOrCreateFile) { for (int i = 0; i < 128; ++i) {d1[i] = 1;} ASSERT_EQ(128, f1.write(128, d1)); - FS::File2 f2 = fs.getOrCreateFile("test.txt"); + FS::File f2 = fs.getOrCreateFile("test.txt"); ASSERT_EQ(128, f2.getSize()); ASSERT_EQ(128, f2.read(128, d2)); for (uint32_t i = 0; i < 128; ++i) {ASSERT_EQ(d1[i], d2[i]);} } + + +template void dumpStructure(FS& fs, DirIter dirIter) { + + while(true) { + auto h = dirIter.nextUsable(); + if (!h.isValid()) {return;} + std::cout << h.getName() << std::endl; + if (h.isDot()) {continue;} + if (h.isDotDot()) {continue;} + if (h.isDirectory()) { + dumpStructure(fs, DirIter(fs, h)); + } + } + +} + +TEST (TestCreate, subfolders) { + + using BlockDev = AccessHelper; + using FS = FAT32::FS; + + size_t size = 32*1024*1024; + TestDevice dev(size); + BlockDev bDev(dev); + + FS fs(bDev, 0); + fs.setup(size, true); + + + + fs.mkdirs("/test1"); + fs.mkdirs("/test1/1"); + fs.mkdirs("/test1/2"); + fs.mkdirs("/test1/2/3"); + + fs.mkdirs("/test2/1/2/3"); + + fs.mkdirs("/test1/a/b"); + fs.mkdirs("/test2/x/y"); + + fs.getOrCreateFile("/hallo.txt"); + + fs.getOrCreateFile("/test1/x/y/z.txt"); + + fs.getOrCreateFile("/test3/a/bbbb.txt"); + + FAT32::DirHandle dh1a = fs.getHandle("/"); + FAT32::DirHandle dh1b = fs.getHandle(""); + FAT32::DirHandle dh2 = fs.getHandle("/test1"); + FAT32::DirHandle dh3 = fs.getHandle("/test1/1"); + + FS::DirIterator di(fs, 0); + + ASSERT_EQ(2, dh1a.getFirstCluster()); ASSERT_TRUE(dh1a.isValid()); + ASSERT_EQ(2, dh1b.getFirstCluster()); ASSERT_TRUE(dh1b.isValid()); + ASSERT_EQ(3, dh2.getFirstCluster()); ASSERT_TRUE(dh2.isValid()); + ASSERT_EQ(4, dh3.getFirstCluster()); ASSERT_TRUE(dh3.isValid()); + + + dev.toFile("/tmp/ram/subdirs.fat32"); + // mount -t vfat /tmp/ram/subdirs.fat32 /mnt/fat/ && ls -l /mnt/fat/ && umount /mnt/fat + + dumpStructure(fs, fs.getRoot()); + +}