kazu/ESP8266lib
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

worked on FAT stuff and tests

Browse Source
This commit is contained in:
FrankE
2021-02-21 21:04:11 +01:00
parent 4ac72c678f
commit 2e281f6d26
12 changed files with 758 additions and 396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

146
ext/sd/fat32/DirHandle.h Normal file
View File

@@ -0,0 +1,146 @@
#include "Structs.h"
#include "Helper.h"
namespace FAT32 {
/** combine a DirectoryEntry with its absolute location on disk */
class DirHandle {
friend class File;
friend class DirIterator;
template <typename BlockDev> friend class FS;
/** the absolute byte location on the disk where the underlying DirEntry is stored */
AbsPos posOnDisk;
/** the FAT32 entry format */
DirEntry entry;
/** is this a valid handle? */
bool valid;
public:
/** ctor for an invalid handle */
DirHandle() : valid(false) {}
/** ctor for a valid handle */
DirHandle(AbsPos posOnDisk, DirEntry* entry) : posOnDisk(posOnDisk), entry(*entry), valid(true) {}
static DirHandle invalid() {return DirHandle();}
/** is this a valid handle? */
bool isValid() const {return valid;}
/** get the entry's size */
uint32_t getSize() const {return entry.size;}
/** get the entry's name (filename / foldername) */
std::string getName() const {
char buf[16];
// the name
uint8_t pos = 0;
for (uint8_t i = 0; i < 8; ++i) {if (entry.name[i] != ' ') {buf[pos++] = entry.name[i];}}
// the extension (if any, folder have none)
if (entry.ext[0] != ' ') {
buf[pos++] = '.';
for (uint8_t i = 0; i < 3; ++i) {if ( entry.ext[i] != ' ') {buf[pos++] = entry.ext[i];}}
}
// null terminate and return
buf[pos] = 0;
std::string str(buf);
toLower(str);
return str;
}
bool isUnused() const {return entry.name[0] == 0xE5;}
bool isDirectory() const {return entry.attr.bits.directory;}
bool isEndOfDirectory() const {return entry.name[0] == 0x00;}
bool isLongFileName() const {return (entry.attr.raw & 0b1111) == 0b1111;}
bool isDot() const {return entry.name[0] == '.' && entry.name[1] == ' ';}
bool isDotDot() const {return entry.name[0] == '.' && entry.name[1] == '.' && entry.name[2] == ' ';}
ClusterNr getFirstCluster() const {return entry.firstClusterHi<<16 | entry.firstClusterLo<<0;}
bool hasExtension(const char* ext) const {
const size_t len = strlen(ext);
if (len > 3) {return false;}
for (int i = 0; i < len; ++i) {
if (asciitolower(entry.ext[i]) != asciitolower(ext[i])) {return false;}
}
for (int i = len; i < 3; ++i) {
if (entry.ext[i] != ' ') {return false;}
}
return true;
}
private:
void setSize(uint32_t size) {entry.size = size;}
/** this entry is an end-of-directory marker */
void setEndOfDirectory() {
entry.name[0] = 0x00;
}
/** this entry is a directory */
void setDirectory() {
setSize(0);
entry.attr.bits.directory = 1;
}
/** this entry is the '.' pointer (to self) */
void setDot(ClusterNr myClusterNr) {
setSize(0);
entry.attr.bits.directory = 1; // not explicitly pointed out within manual, but e.g. linux needs it
memcpy(entry.name83, ". ", 8+3);
setFirstCluster(myClusterNr);
}
/** this entry is the '.' pointer (to parent) */
void setDotDot(ClusterNr parentClusterNr) {
setSize(0);
entry.attr.bits.directory = 1; // not explicitly pointed out within manual, but e.g. linux needs it
memcpy(entry.name83, ".. ", 8+3);
setFirstCluster(parentClusterNr);
}
void setFirstCluster(ClusterNr nr) {entry.firstClusterHi = (uint16_t)(nr << 16); entry.firstClusterLo = (uint16_t)(nr << 0);}
void setName(const std::string& str) {
// "zero" fill name and extension (0x20)
memset(entry.name83, ' ', 11);
auto pos = str.find('.');
// name
for (size_t i = 0; i < min(8, min(pos, str.length())); ++i) {
if (i >= 8) {break;}
entry.name[i] = str[i];
}
// extension (if any, folders have none)
if (pos != std::string::npos) {
uint32_t extLen = min(3, str.length() - pos - 1);
for (size_t i = 0; i < extLen; ++i) {
if (i >= 3) {break;}
entry.ext[i] = str[i+pos+1];
}
}
}
};
}

130
ext/sd/fat32/DirHelper.h Normal file
View File

@@ -0,0 +1,130 @@
#pragma once
class DirHelper {
static constexpr const char* NAME = "FAT32_DirH";
public:
enum class CreateType {
NONE,
DIR,
FILE
};
/**
* get (or allocate, if needed) a new DirEntry with the given name absolute name
* if the entry is allocated, it is UNTYPED
* name: e.g. /path1/path2/path3 or /path1/path2/file.txt
*/
static DirHandle getOrCreate(FS& fs, ClusterNr dirStartCluster, const std::string& absName, CreateType ct) {
// determine the local name
Split s = split(absName);
{ // search for a matching existing entry
DirIterator di(fs, dirStartCluster);
while(true) {
DirHandle h = di.nextUsable();
if (!h.isValid()) {break;}
// found a matching entry
const std::string n = h.getName();
if (n == s.first) {
// are we done yet?
if (s.hasMore()) {
const ClusterNr subDirStartCluster = h.getFirstCluster();
return getOrCreate(fs, subDirStartCluster, s.next, ct);
} else {
return h;
}
}
}
}
// no matching entry found
if (ct == CreateType::NONE) {return DirHandle::invalid();}
{ // allocate a new DirEntry within the current directory
DirIterator di(fs, dirStartCluster);
DirHandle h = di.nextFree();
h.setName(s.first);
h.setSize(0);
h.setFirstCluster(0);
// if this is a sub-directory, allocate a new cluster for it
if (s.hasMore() || ct == CreateType::DIR) {
// allocate a cluster where the directory contents are to be stored
ClusterNr freeCluster = fs.allocFreeCluster(0);
h.setFirstCluster(freeCluster);
h.setDirectory();
fs.write(h);
// ensure the new cluster is initialized with all zeros (this is also equal to an immediate END_OF_DIRECTORY marker in the beginning
fs.zeroOutCluster(freeCluster);
DirIterator di2(fs, freeCluster);
// create the "." entry (pointer to current directory)
DirHandle h2 = di2.next();
h2.setDot(freeCluster);
fs.write(h2);
// create the ".." entry (pointer to parent directory)
DirHandle h3 = di2.next();
h3.setDotDot(dirStartCluster);
fs.write(h3);
// more path to process?
if (s.hasMore()) {
return getOrCreate(fs, freeCluster, s.next, ct);
} else {
return h;
}
} else {
fs.write(h);
return h;
}
}
}
private:
struct Split {
std::string first;
std::string next;
Split(const std::string& first, const std::string& next) : first(first), next(next) {}
bool hasMore() const {return !next.empty();}
};
/** split "/path1/path2/path3/file.txt" into "path1" and "path2/path3/file.txt" */
static Split split(std::string absPath) {
if (absPath[0] == '/') {
absPath = absPath.substr(1);
}
std::size_t pos = absPath.find('/');
if (pos == std::string::npos) {
return Split(absPath, "");
} else {
return Split(absPath.substr(0, pos), absPath.substr(pos));
}
}
};

74
ext/sd/fat32/DirIterator.h
View File

@@ -15,63 +15,73 @@ public:
Log::addInfo(NAME, "init @ Cluster %d", curCluster);
// ".." folders point to cluster 0 if they point to the root folder -> adjust here
if (clusterNr == 0) {
clusterNr = fs.desc.rootDirFirstCluster;
}
// read the first sector in the first cluster
read(curCluster, 0);
}
DirIterator(FS& fs, DirHandle dh) : DirIterator(fs, dh.getFirstCluster()) {
}
/** get the next usable entry within the current directory */
DirEntryAt nextUsable() {
DirHandle nextUsable() {
while(true) {
DirEntryAt dea = next(false);
DirHandle h = next(false);
// check it
if (!dea.isValid()) {return DirEntryAt::invalid();}
if (dea.isLongFileName()) {continue;}
if (dea.isUnused()) {continue;}
if (dea.isEndOfDirectory()) {return DirEntryAt::invalid();}
if (!h.isValid()) {return DirHandle::invalid();}
if (h.isLongFileName()) {continue;}
if (h.isUnused()) {continue;}
if (h.isEndOfDirectory()) {return DirHandle::invalid();}
// usable!
return dea;
return h;
}
}
/** get (or create) a free entry within the current directory */
DirEntryAt nextFree() {
DirHandle nextFree() {
while(true) {
DirEntryAt dea = next(true);
DirHandle h = next(true);
// check it
if (!dea.isValid()) {return DirEntryAt::invalid();}
if (!h.isValid()) {return DirHandle::invalid();}
if (dea.isUnused()) {
if (h.isUnused()) {
// switch from "unused" to something usable
//dea.setName("NEW.ONE");
//dea.entry.attr.raw = 0;
//h.setName("NEW.ONE");
//h.entry.attr.raw = 0;
//return dea;
}
if (dea.isEndOfDirectory()) {
if (h.isEndOfDirectory()) {
// add a new EndOfDirectory entry afterwards
DirEntryAt dea2 = next(true);
dea2.setEndOfDirectory();
fs.write(dea2);
DirHandle h2 = next(true);
h2.setEndOfDirectory();
fs.write(h2);
// switch from "end of directory" to something usable
dea.setName("NEW.ONE");
dea.entry.attr.raw = 0;
return dea;
h.setName("NEW.ONE");
h.entry.attr.raw = 0;
return h;
}
@@ -80,15 +90,19 @@ public:
}
private:
DirEntryAt cur() {
AbsPos pos = fs.clusterToAbsPos(curCluster) + (curSectorInCluster * fs.desc.bytesPerSector) + (curEntryInSector * sizeof(DirEntry));
DirEntry* dirEntry = reinterpret_cast<DirEntry*>(buf + (curEntryInSector * sizeof(DirEntry)));
return DirEntryAt(pos, dirEntry);
DirHandle next() {
return next(false);
}
DirEntryAt next(bool allocIfNeeded) {
private:
DirHandle cur() {
AbsPos pos = fs.clusterToAbsPos(curCluster) + (curSectorInCluster * fs.desc.bytesPerSector) + (curEntryInSector * sizeof(DirEntry));
DirEntry* dirEntry = reinterpret_cast<DirEntry*>(buf + (curEntryInSector * sizeof(DirEntry)));
return DirHandle(pos, dirEntry);
}
DirHandle next(bool allocIfNeeded) {
while(true) {
@@ -110,7 +124,7 @@ private:
// do not alloc new entries? -> done here
if (!allocIfNeeded) {
return DirEntryAt::invalid();
return DirHandle::invalid();
} else {
curCluster = fs.allocFreeCluster(curCluster);
}
@@ -129,10 +143,10 @@ private:
}
// the current entry
DirEntryAt dea = cur();
DirHandle h = cur();
++curEntryInSector;
return dea;
return h;
}

172
ext/sd/fat32/FS.h
View File

@@ -5,6 +5,7 @@
#include <vector>
#include "Structs.h"
#include "DirHandle.h"
// https://www.pjrc.com/tech/8051/ide/fat32.html
namespace FAT32 {
@@ -23,41 +24,68 @@ namespace FAT32 {
#include "FreeClusterIterator.h"
FreeClusterIterator fci;
#include "UsedSpaceChecker.h"
public:
#include "File.h"
#include "File2.h"
#include "DirIterator.h"
#include "DirHelper.h"
/** ctor with the absolute offset addr (in bytes) */
FS(BlockDev& dev, AbsOffset offset) : dev(dev), offset(offset), fci(*this) {
init();
}
void setup(uint32_t totalSize) {
/**
* setup a new filesystem with the given parameters
* killFAT = zero both file allocation tables (reset all cluster numbers)
*/
void setup(uint32_t totalSize, const bool killFAT) {
uint8_t buf[512] = {0};
buf[0x1FE] = 0x55;
buf[0x1FF] = 0xAA;
FSHeader* header = (FSHeader*) buf;
header->bytesPerSector = 512;
header->numberOfFATs = 2;
header->numReservedSectors = 32;
header->rootDirFirstCluster = 2;
header->mediaDescriptor = 0xF8; // hard disk
header->sectorsInPartition = totalSize / 512;
header->sectorsPerCluster = 8;
header->sectorsPerFAT = 64; // 8 MB
const uint32_t sectorsPerCluster = 8;
const uint32_t sizePerCluster = 512 * sectorsPerCluster;
const uint32_t sectorsPerFAT = totalSize / sizePerCluster * sizeof(ClusterNr) / 512;
FSHeader* header = (FSHeader*) buf;
header->bytesPerSector = 512; // always
header->numberOfFATs = 2; // always
header->numReservedSectors = 32; // always
header->rootDirFirstCluster = 2; // always
header->mediaDescriptor = 0xF8; // hard disk
header->sectorsInPartition = totalSize / 512;
header->sectorsPerCluster = sectorsPerCluster; // 4096 byte clusters
header->sectorsPerFAT = sectorsPerFAT;
// write the header to the underlying device
dev.write(offset, 512, (const uint8_t*)header);
// now update the internal state based on the newly created FS header...
init();
// mark the root-dir cluster as used
setNextCluster(2, END_OF_CLUSTERS);
if (killFAT) {
// zero-out FAT1
uint8_t zeros[512] = {0};
AbsPos pos = tmp.startOfFAT1;
for (uint32_t i = 0; i < header->sectorsPerFAT; ++i) {
dev.write(pos + i * 512, 512, zeros);
}
// ...and mark the root-dir cluster as used
setNextCluster(2, END_OF_CLUSTERS);
// set the first entry within the root-dir to denote the end-of-the-root-dir (no entries at all)
DirIterator di = getRoot();
DirHandle dh = di.next();
dh.setEndOfDirectory();
write(dh);
}
}
@@ -66,34 +94,64 @@ namespace FAT32 {
return valid;
}
/** get the total size of the filesystem (including FATs etc.) */
uint32_t getSize() const {
return desc.sectorsPerFAT * desc.bytesPerSector / sizeof(ClusterNr) * tmp.bytesPerCluster;
}
/** get the size that is actually usable for data (total size - required overhead) */
uint32_t getSizeForData() const {
return getSize() - (desc.numReservedSectors * desc.bytesPerSector) - (desc.numberOfFATs * desc.sectorsPerFAT * desc.bytesPerSector);
}
/** get the number of used data bytes */
uint32_t getSizeUsed() const {
return UsedSpaceChecker::getNumUsedClusters(*this) * tmp.bytesPerCluster;
}
/** get an iterator for the root directory */
DirIterator getRoot() {
return DirIterator(*this, desc.rootDirFirstCluster);
return DirIterator(*this, 2);
}
/** open the given file for reading*/
File open(const DirEntryAt& dea) {
return File(*this, dea.getSize(), dea.getFirstCluster());
/** open the given file */
File open(const DirHandle& h) {
return File(*this, h);
}
/** open the given file for reading */
File2 open2(const DirEntryAt& dea) {
return File2(*this, dea);
/** create the given absolute folder */
DirHandle mkdirs(const char* absName) {
DirHandle h = DirHelper::getOrCreate(*this, 2, absName, DirHelper::CreateType::DIR);
return h;
}
/** get the entry with the given name */
DirHandle getHandle(const char* absName) {
std::string tmp(absName);
if (tmp == "/" || tmp == "") {
DirEntry dummy;
DirHandle dh(0, &dummy);
dh.setFirstCluster(desc.rootDirFirstCluster);
return dh;
} else {
DirHandle h = DirHelper::getOrCreate(*this, 2, absName, DirHelper::CreateType::NONE);
return h;
}
}
/** get or create a file with the given name */
File2 getOrCreateFile(const char* name) {
File getOrCreateFile(const char* absName) {
DirEntryAt dea = getDirEntry(name, true);
DirHandle h = DirHelper::getOrCreate(*this, 2, absName, DirHelper::CreateType::FILE);
// new file -> allocate the first cluster
if (dea.getFirstCluster() == 0) {
if (h.getFirstCluster() == 0) {
ClusterNr firstCluster = allocFreeCluster(0);
dea.setFirstCluster(firstCluster);
write(dea);
h.setFirstCluster(firstCluster);
write(h);
}
return File2(*this, dea);
return File(*this, h);
}
@@ -117,49 +175,20 @@ namespace FAT32 {
valid = (desc.bytesPerSector == 512) && (desc.numberOfFATs == 2) && (getU16(&buf[0x1FE]) == 0xAA55);
tmp.bytesPerCluster = desc.sectorsPerCluster * desc.bytesPerSector;
tmp.startOfFAT = offset + (desc.numReservedSectors * desc.bytesPerSector);
tmp.startOfFAT1 = offset + (desc.numReservedSectors * desc.bytesPerSector);
tmp.startOfFAT2 = tmp.startOfFAT1 + desc.sectorsPerFAT * desc.bytesPerSector;
tmp.startOfFirstDataCluster = offset + (desc.numReservedSectors * desc.bytesPerSector) + (desc.numberOfFATs * desc.sectorsPerFAT * desc.bytesPerSector);
tmp.startOfFirstRootDirCluster = clusterToAbsPos(desc.rootDirFirstCluster);
tmp.entriesPerFAT = desc.sectorsPerFAT * desc.bytesPerSector / sizeof(ClusterNr);
tmp.dirEntriesPerSector = desc.bytesPerSector / sizeof(DirEntry);
Log::addInfo(NAME, "Bytes/Sector: %d, Sector/Cluster: %d, FATs: %d, RootDir: %d", desc.bytesPerSector, desc.sectorsPerCluster, desc.numberOfFATs, desc.rootDirFirstCluster);
}
/** get (or create, if needed) a DirEntry with the given name */
DirEntryAt getDirEntry(const char* name, bool createIfNeeded) {
// TODO: support for sub folders)
// start at the root folder
ClusterNr dirStartCluster = 2;
{ // search for a matching existing entry
DirIterator di(*this, dirStartCluster);
while(true) {
DirEntryAt dea = di.nextUsable();
if (!dea.isValid()) {break;}
if (dea.getName() == name) {return dea;}
}
}
// no matching entry found
if (!createIfNeeded) {return DirEntryAt::invalid();}
{ // allocate a new DirEntry for the file
DirIterator di(*this, dirStartCluster);
DirEntryAt dea = di.nextFree();
dea.setName(name);
dea.setSize(0);
dea.setFirstCluster(0);
return dea;
}
}
/** determine the ClusterNr following the given ClusterNr */
ClusterNr getNextCluster(const ClusterNr clusterNr) {
const AbsPos pos = tmp.startOfFAT + (clusterNr * sizeof(ClusterNr));
ClusterNr getNextCluster(const ClusterNr clusterNr) const {
const AbsPos pos = tmp.startOfFAT1 + (clusterNr * sizeof(ClusterNr));
ClusterNr next = 0;
dev.read(pos, sizeof(ClusterNr), reinterpret_cast<uint8_t*>(&next));
Log::addInfo(NAME, "getNextCluster(%d) -> %d", clusterNr, next);
@@ -168,8 +197,8 @@ namespace FAT32 {
/** set the ClusterNr following clusterNr */
void setNextCluster(const ClusterNr clusterNr, const ClusterNr next) {
const AbsPos pos = tmp.startOfFAT + (clusterNr * sizeof(ClusterNr));
dev.write(pos, sizeof(ClusterNr), reinterpret_cast<const uint8_t*>(&next));
const AbsPos pos = tmp.startOfFAT1 + (clusterNr * sizeof(ClusterNr));
dev.write(pos, sizeof(ClusterNr), reinterpret_cast<const uint8_t*>(&next.val));
Log::addInfo(NAME, "setNextCluster(%d) -> %d", clusterNr, next);
}
@@ -188,9 +217,18 @@ namespace FAT32 {
return newCluster;
}
/** write the given DirEntry back to disk */
void write(const DirEntryAt& dea) {
dev.write(dea.posOnDisk, sizeof(DirEntry), (const uint8_t*)&dea.entry);
/** write all zeros to the given cluster */
void zeroOutCluster(ClusterNr nr) {
uint8_t zeros[512] = {0};
AbsPos pos = clusterToAbsPos(nr);
for (uint8_t i = 0; i < desc.sectorsPerCluster; ++i) {
dev.write(pos + i * 512, 512, zeros);
}
}
/** write the given DirHandle back to disk */
void write(const DirHandle& h) {
dev.write(h.posOnDisk, sizeof(DirEntry), (const uint8_t*)&h.entry);
}
};

132
ext/sd/fat32/File.h
View File

@@ -4,25 +4,36 @@ class File {
static constexpr const uint32_t F_EOF = 0xFFFFFFFF;
FS& fs;
uint32_t totalSize;
DirHandle h;
uint32_t curAbsPos = 0; // position withithin the whole file
ClusterNr curCluster = 0; // cluster we are currently reading
uint32_t curAbsPos = 0; // absolute position withithin the file
uint32_t posInCluster = 0; // position within the current cluster
uint32_t iCurCluster = 0; // current cluster index (within list of clusters)
std::vector<ClusterNr> clusters; // all clusters the file uses
public:
File(FS& fs, uint32_t size, ClusterNr firstCluster) : fs(fs), totalSize(size), curCluster(firstCluster) {
Log::addInfo(NAME, "init @ cluster %d", firstCluster);
File(FS& fs, const DirHandle& h) : fs(fs), h(h) {
Log::addInfo(NAME, "init @ cluster %d", getFirstCluster());
getAllClusters(h.getFirstCluster());
}
/** the file's size */
uint32_t getSize() const {return totalSize;}
/** the file's USED size */
uint32_t getSize() const {return h.getSize();}
uint32_t read(uint32_t size, uint8_t* dst, std::function<void(int)> callback) {
/** the file's ALLOCATED size */
uint32_t getAllocatedSize() const {return clusters.size() * fs.tmp.bytesPerCluster;}
/** the file's first cluster */
ClusterNr getFirstCluster() const {return h.getFirstCluster();}
/** get the file's name */
std::string getName() const {return h.getName();}
/** read x bytes from the file */
uint32_t read(uint32_t size, uint8_t* dst) {
Log::addInfo(NAME, "read %d bytes", size);
@@ -37,27 +48,75 @@ public:
remaining -= read;
dst += read;
totalRead += read;
callback(totalRead*100/size);
}
return totalRead;
}
/* write the given data into the file */
uint32_t write(uint32_t size, const uint8_t* src) {
Log::addInfo(NAME, "write %d bytes", size);
uint32_t remaining = size;
uint32_t totalWritten = 0;
while(remaining) {
const uint32_t written = _write(remaining, src);
remaining -= written;
src += written;
totalWritten += written;
//callback(totalWritten*100/size);
}
// update the file header (filesize might have changed)
fs.write(h);
return totalWritten;
}
private:
void setSize(uint32_t size) {h.setSize(size);}
/** fetch the list of all clusters the file is using */
void getAllClusters(ClusterNr startCluster) {
// initial cluster
clusters.push_back(startCluster);
// all following clusters
while(true) {
startCluster = fs.getNextCluster(startCluster);
if (!startCluster.isEndOfClusters()) {
clusters.push_back(startCluster);
} else {
break;
}
}
// debug
//char buf[1024];
//char* dst = buf; dst += sprintf(dst, "clusters: ");
//for (ClusterNr nr : clusters) {dst += sprintf(dst, "%d ", nr);}
//Log::addInfo(NAME, buf);
}
int32_t _read(uint32_t readSize, uint8_t* dst) {
// EOF reached?
if (curAbsPos >= totalSize) {
if (curAbsPos >= getSize()) {
return F_EOF;
}
// end of current cluster reached? -> determine the next one
// end of current cluster reached? -> switch to the next one
if (posInCluster == fs.tmp.bytesPerCluster) {
curCluster = fs.getNextCluster(curCluster);
++iCurCluster;
posInCluster = 0;
Log::addInfo(NAME, "next cluster %d", curCluster);
Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]);
}
// how many bytes are left in the current cluster?
@@ -66,7 +125,7 @@ private:
// determine how many bytes to read
const uint32_t toRead = std::min(remainingInCluster, readSize);
const uint32_t offset = fs.clusterToAbsPos(curCluster) + posInCluster;
const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster;
const uint32_t read = fs.dev.read(offset, toRead, dst);
posInCluster += read;
curAbsPos += read;
@@ -75,4 +134,45 @@ private:
}
int32_t _write(uint32_t writeSize, const uint8_t* src) {
// do we need to append more free sectors to the end of the file?
if (curAbsPos >= getAllocatedSize()) {
if (clusters.empty()) {
const ClusterNr newCluster = fs.allocFreeCluster(0);
clusters.push_back(newCluster);
Log::addInfo(NAME, "allocFirstCluster: %d", newCluster);
} else {
const ClusterNr prevCluster = clusters.back();
const ClusterNr newCluster = fs.allocFreeCluster(prevCluster);
clusters.push_back(newCluster);
Log::addInfo(NAME, "allocNextCluster(%d): %d", prevCluster, newCluster);
}
}
// end of current cluster reached? -> switch to the next one
if (posInCluster == fs.tmp.bytesPerCluster) {
++iCurCluster;
posInCluster = 0;
Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]);
}
// how many bytes are left in the current cluster?
const uint32_t remainingInCluster = fs.tmp.bytesPerCluster - posInCluster;
// determine how many bytes to write
const uint32_t toWrite = std::min(remainingInCluster, writeSize);
const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster;
const uint32_t written = fs.dev.write(offset, toWrite, src);
posInCluster += written;
curAbsPos += written;
// adjust the number of bytes used
if (this->getSize() < curAbsPos) {this->setSize(curAbsPos);}
return written;
}
};

178
ext/sd/fat32/File2.h
View File

@@ -1,178 +0,0 @@
class File2 {
static constexpr const char* NAME = "FAT32_File2";
static constexpr const uint32_t F_EOF = 0xFFFFFFFF;
FS& fs;
DirEntryAt dea;
uint32_t curAbsPos = 0; // absolute position withithin the file
uint32_t posInCluster = 0; // position within the current cluster
uint32_t iCurCluster = 0; // current cluster index (within list of clusters)
std::vector<ClusterNr> clusters; // all clusters the file uses
public:
File2(FS& fs, const DirEntryAt& dea) : fs(fs), dea(dea) {
Log::addInfo(NAME, "init @ cluster %d", getFirstCluster());
getAllClusters(dea.getFirstCluster());
}
/** the file's USED size */
uint32_t getSize() const {return dea.getSize();}
/** the file's ALLOCATED size */
uint32_t getAllocatedSize() const {return clusters.size() * fs.tmp.bytesPerCluster;}
/** the file's first cluster */
ClusterNr getFirstCluster() const {return dea.getFirstCluster();}
/** get the file's name */
std::string getName() const {return dea.getName();}
/** read x bytes from the file */
uint32_t read(uint32_t size, uint8_t* dst) {
Log::addInfo(NAME, "read %d bytes", size);
uint32_t remaining = size;
uint32_t totalRead = 0;
while(remaining) {
const uint32_t read = _read(remaining, dst);
if (read == F_EOF) {
Log::addInfo(NAME, "EOF"); break;
}
remaining -= read;
dst += read;
totalRead += read;
}
return totalRead;
}
/* write the given data into the file */
uint32_t write(uint32_t size, const uint8_t* src) {
Log::addInfo(NAME, "write %d bytes", size);
uint32_t remaining = size;
uint32_t totalWritten = 0;
while(remaining) {
const uint32_t written = _write(remaining, src);
remaining -= written;
src += written;
totalWritten += written;
//callback(totalWritten*100/size);
}
// update the file header (filesize might have changed)
fs.write(dea);
return totalWritten;
}
private:
void setSize(uint32_t size) {dea.setSize(size);}
/** fetch the list of all clusters the file is using */
void getAllClusters(ClusterNr startCluster) {
// initial cluster
clusters.push_back(startCluster);
// all following clusters
while(true) {
startCluster = fs.getNextCluster(startCluster);
if (!startCluster.isEndOfClusters()) {
clusters.push_back(startCluster);
} else {
break;
}
}
// debug
//char buf[1024];
//char* dst = buf; dst += sprintf(dst, "clusters: ");
//for (ClusterNr nr : clusters) {dst += sprintf(dst, "%d ", nr);}
//Log::addInfo(NAME, buf);
}
int32_t _read(uint32_t readSize, uint8_t* dst) {
// EOF reached?
if (curAbsPos >= getSize()) {
return F_EOF;
}
// end of current cluster reached? -> switch to the next one
if (posInCluster == fs.tmp.bytesPerCluster) {
++iCurCluster;
posInCluster = 0;
Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]);
}
// how many bytes are left in the current cluster?
const uint32_t remainingInCluster = fs.tmp.bytesPerCluster - posInCluster;
// determine how many bytes to read
const uint32_t toRead = std::min(remainingInCluster, readSize);
const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster;
const uint32_t read = fs.dev.read(offset, toRead, dst);
posInCluster += read;
curAbsPos += read;
return read;
}
int32_t _write(uint32_t writeSize, const uint8_t* src) {
// do we need to append more free sectors to the end of the file?
if (curAbsPos >= getAllocatedSize()) {
if (clusters.empty()) {
const ClusterNr newCluster = fs.allocFreeCluster(0);
clusters.push_back(newCluster);
Log::addInfo(NAME, "allocFirstCluster: %d", newCluster);
} else {
const ClusterNr prevCluster = clusters.back();
const ClusterNr newCluster = fs.allocFreeCluster(prevCluster);
clusters.push_back(newCluster);
Log::addInfo(NAME, "allocNextCluster(%d): %d", prevCluster, newCluster);
}
}
// end of current cluster reached? -> switch to the next one
if (posInCluster == fs.tmp.bytesPerCluster) {
++iCurCluster;
posInCluster = 0;
Log::addInfo(NAME, "next cluster [%d] %d", iCurCluster, clusters[iCurCluster]);
}
// how many bytes are left in the current cluster?
const uint32_t remainingInCluster = fs.tmp.bytesPerCluster - posInCluster;
// determine how many bytes to write
const uint32_t toWrite = std::min(remainingInCluster, writeSize);
const uint32_t offset = fs.clusterToAbsPos(clusters[iCurCluster]) + posInCluster;
const uint32_t written = fs.dev.write(offset, toWrite, src);
posInCluster += written;
curAbsPos += written;
// adjust the number of bytes used
if (this->getSize() < curAbsPos) {this->setSize(curAbsPos);}
return written;
}
};

8
ext/sd/fat32/FreeClusterIterator.h
View File

@@ -1,6 +1,10 @@
#pragma once
/** helper class to iterate all free clusters of the Filesystem */
/**
* helper class to iterate all free clusters of the Filesystem.
* starts at cluster number 2 and then iterates over all subsequent clusters
* stopping at the free ones
*/
class FreeClusterIterator {
FS& fs;
@@ -15,6 +19,8 @@ public:
/** get the next free cluster that is available */
ClusterNr next() {
// TODO: end of filesystem reached
while(true) {
const ClusterNr tmp = fs.getNextCluster(cur);
if (tmp.isFree()) {return cur;}

13
ext/sd/fat32/Helper.h Normal file
View File

@@ -0,0 +1,13 @@
#pragma once
#include <string>
#include <algorithm>
static char asciitolower(char in) {
if (in <= 'Z' && in >= 'A') {return in - ('Z' - 'z');}
return in;
}
static void toLower(std::string& str) {
std::transform(str.begin(), str.end(), str.begin(), asciitolower);
}

123
ext/sd/fat32/Structs.h
View File

@@ -6,12 +6,12 @@
namespace FAT32 {
static inline uint16_t getU8(const uint8_t* src) {return src[0];}
static inline uint16_t getU16(const uint8_t* src) {return src[0]<<0 | src[1]<<8;}
static inline uint32_t getU32(const uint8_t* src) {return src[0]<<0 | src[1]<<8 | src[2]<<16 | src[3]<<24;}
static inline size_t min(size_t a, size_t b) {return (a<b) ? a : b;}
static inline uint16_t getU8(const uint8_t* src) {return src[0];}
static inline uint16_t getU16(const uint8_t* src) {return src[0]<<0 | src[1]<<8;}
static inline uint32_t getU32(const uint8_t* src) {return src[0]<<0 | src[1]<<8 | src[2]<<16 | src[3]<<24;}
static inline size_t min(size_t a, size_t b) {return (a<b) ? a : b;}
/** describes a uint32_t Cluster Number and typical helper functions */
struct ClusterNr {
uint32_t val;
@@ -35,10 +35,10 @@ static inline size_t min(size_t a, size_t b) {return (a<b) ? a : b;}
};
const ClusterNr END_OF_CLUSTERS = 0xFFFFFFF;
const ClusterNr END_OF_CLUSTERS = 0x0FFFFFF8;
/** header within the first 512 bytes */
/** FAT32: header within the first 512 bytes of the Filesystem */
struct FSHeader { // https://www.easeus.com/resource/fat32-disk-structure.htm
uint8_t dummy1[11];
uint16_t bytesPerSector; // @0x0B;
@@ -49,28 +49,12 @@ static inline size_t min(size_t a, size_t b) {return (a<b) ? a : b;}
uint8_t mediaDescriptor; // @0x15;
uint8_t dummy3[10];
uint32_t sectorsInPartition; // @0x20;
uint32_t sectorsPerFAT; // @0x24;
uint32_t sectorsPerFAT; // @0x24; // home many sectors does each of the 2 FATs contain?
uint8_t dummy4[4];
uint32_t rootDirFirstCluster; // @0x2C
} __attribute__((packed));
struct FSDesc {
uint16_t bytesPerSector;
uint8_t sectorsPerCluster;
uint16_t numReservedSectors;
uint8_t numberOfFATs;
uint32_t sectorsPerFAT;
ClusterNr rootDirFirstCluster;
};
struct Precomputed {
uint32_t bytesPerCluster;
AbsPos startOfFAT; // absolute byte offset where the FAT begins
AbsPos startOfFirstDataCluster; // absolute byte offset where the first cluster is
AbsPos startOfFirstRootDirCluster; // absolute byte offset where the first root dir cluster is
uint32_t dirEntriesPerSector; // number of directory entries that fit into a sector
};
/** FAT32: attributes of a DirEntry */
union Attributes {
uint8_t raw;
struct {
@@ -85,10 +69,15 @@ static inline size_t min(size_t a, size_t b) {return (a<b) ? a : b;}
} bits;
} __attribute__((packed));
/** structure for a DirectoryEntry as defined in the FAT standard */
/** FAT32: structure for a DirectoryEntry as defined in the FAT standard */
struct DirEntry {
unsigned char name[8];
unsigned char ext[3];
union {
unsigned char name83[8+3];
struct {
unsigned char name[8];
unsigned char ext[3];
} __attribute__((packed));
} __attribute__((packed));
Attributes attr;
uint8_t dummy1[8];
uint16_t firstClusterHi;
@@ -97,67 +86,29 @@ static inline size_t min(size_t a, size_t b) {return (a<b) ? a : b;}
uint32_t size; // file size in bytes
} __attribute__((packed));
/** combine a DirectoryEntry with its absolute location on disk */
struct DirEntryAt {
AbsPos posOnDisk;
DirEntry entry;
bool valid;
/** ctor for an invalid entry */
DirEntryAt() : valid(false) {}
/** ctor for a valid entry */
DirEntryAt(AbsPos posOnDisk, DirEntry* entry) : posOnDisk(posOnDisk), entry(*entry), valid(true) {}
static DirEntryAt invalid() {return DirEntryAt();}
/** is this a valid entry? */
bool isValid() const {return valid;}
uint32_t getSize() const {return entry.size;}
void setSize(uint32_t size) {entry.size = size;}
bool isUnused() const {return entry.name[0] == 0xE5;}
bool isDirectory() const {return entry.attr.bits.directory;}
bool isEndOfDirectory() const {return entry.name[0] == 0x00;}
bool isLongFileName() const {return (entry.attr.raw & 0b1111) == 0b1111;}
void setEndOfDirectory() {entry.name[0] = 0x00;}
ClusterNr getFirstCluster() const {return entry.firstClusterHi<<16 | entry.firstClusterLo<<0;}
void setFirstCluster(ClusterNr nr) {entry.firstClusterHi = (uint16_t)(nr << 16); entry.firstClusterLo = (uint16_t)(nr << 0);}
std::string getName() const {
char buf[16];
uint8_t pos = 0;
for (uint8_t i = 0; i < 8; ++i) {if (entry.name[i] != ' ') {buf[pos++] = entry.name[i];}}
buf[pos++] = '.';
for (uint8_t i = 0; i < 3; ++i) {if ( entry.ext[i] != ' ') {buf[pos++] = entry.ext[i];}}
buf[pos] = 0;
return std::string(buf);
}
void setName(const std::string& str) {
// "zero" fill name and extension
memset(entry.name, ' ', 8);
memset(entry.ext, ' ', 3);
auto pos = str.find('.');
for (size_t i = 0; i < min(pos, str.length()); ++i) {
if (i >= 8) {break;}
entry.name[i] = str[i];
}
for (size_t i = 0; i < min(pos, str.length()); ++i) {
if (i >= 3) {break;}
entry.ext[i] = str[i+pos+1];
}
}
/** only the important parts of above header (to save space) */
struct FSDesc {
uint16_t bytesPerSector;
uint8_t sectorsPerCluster;
uint16_t numReservedSectors;
uint8_t numberOfFATs;
uint32_t sectorsPerFAT;
ClusterNr rootDirFirstCluster;
};
/** additional pre-computed values, to reduce calculations */
struct Precomputed {
uint32_t bytesPerCluster;
AbsPos startOfFAT1; // absolute byte offset where the FAT1 begins
AbsPos startOfFAT2; // absolute byte offset where the FAT2 begins
AbsPos startOfFirstDataCluster; // absolute byte offset where the first cluster is
AbsPos startOfFirstRootDirCluster; // absolute byte offset where the first root dir cluster is
uint32_t entriesPerFAT; // number of entries (cluster pointers) within each FAT
uint32_t dirEntriesPerSector; // number of directory entries that fit into a sector
};
}

25
ext/sd/fat32/UsedSpaceChecker.h Normal file
View File

@@ -0,0 +1,25 @@
#pragma once
/**
* helper class to determine all clusters whice are used.
* starts at cluster number 2 and then iterates over all subsequent clusters
*/
class UsedSpaceChecker {
public:
/** get the number of used clusters */
static uint32_t getNumUsedClusters(const FS& fs) {
uint32_t numUsed = 0;
for (ClusterNr nr = 2; nr < fs.tmp.entriesPerFAT; ++nr) {
ClusterNr next = fs.getNextCluster(nr);
if (!next.isFree()) {++numUsed;}
}
return numUsed;
}
};

14
ext/sd/main.cpp
View File

@@ -112,11 +112,11 @@ int main(int argc, char** argv) {
FAT32FS::DirIterator dir = fat.getRoot();
while(false) {
FAT32::DirEntryAt dea = dir.nextUsable();
if (!dea.isValid()) {break;}
FAT32::DirHandle h = dir.nextUsable();
if (!h.isValid()) {break;}
if (1==0) {
FAT32FS::File2 f = fat.open2(dea);
FAT32FS::File f = fat.open(h);
if (1==0) {
uint8_t* bufff = (uint8_t*) malloc(1024*1024);
@@ -137,11 +137,11 @@ int main(int argc, char** argv) {
//FAT32::DirEntryAt root = fat.getRoot().cur();
FAT32FS::File2 file = fat.getOrCreateFile("tmp1.txt");
// //FAT32::DirEntryAt root = fat.getRoot().cur();
// FAT32FS::File file = fat.getOrCreateFile("tmp1.txt");
uint8_t src[128];
file.write(128, src);
// uint8_t src[128];
// file.write(128, src);
// diff /tmp/ram/TETRIS.GB /apps/workspace/gbemu/tests/tetris.gb
// diff /tmp/ram/KIRBY1.GB /apps/workspace/gbemu/tests/Kirby\'s\ Dream\ Land\ \(USA\,\ Europe\).gb

139
ext/sd/tests/TestCreate.cpp
View File

@@ -11,7 +11,7 @@
TEST(TestCreat, structure) {
TEST(TestCreate, structure) {
FAT32::FSHeader header;
@@ -35,7 +35,7 @@ TEST (TestCreat, writeRead) {
TestDevice dev(size);
BlockDev bDev(dev);
FS fs(bDev, 0);
fs.setup(size);
fs.setup(size, true);
for (int i = 0; i < 64; ++i) {
@@ -45,7 +45,7 @@ TEST (TestCreat, writeRead) {
const size_t sizeA = (size/4096+1) * 4096;
std::cout << name << " - " << size << std::endl;
FAT32::FS<BlockDev>::File2 f = fs.getOrCreateFile(name);
FAT32::FS<BlockDev>::File f = fs.getOrCreateFile(name);
uint8_t* data = (uint8_t*)malloc(128 + i * 512);
for (uint32_t j = 0; j < size; ++j) {data[j] = j;}
uint32_t written = f.write(size, data);
@@ -73,10 +73,10 @@ TEST (TestCreat, writeRead) {
const size_t size = 128 + i * 512;
const size_t sizeA = (size/4096+1) * 4096;
FAT32::DirEntryAt dea = di.nextUsable();
FS::File2 f = fs.open2(dea);
FAT32::DirHandle h = di.nextUsable();
FS::File f = fs.open(h);
ASSERT_EQ(name, dea.getName());
ASSERT_EQ(name, h.getName());
ASSERT_EQ(name, f.getName());
ASSERT_EQ(size, f.getSize());
ASSERT_EQ(sizeA, f.getAllocatedSize());
@@ -96,7 +96,46 @@ TEST (TestCreat, writeRead) {
}
TEST (TestCreate, getOrCreateFile) {
TEST (TestCreat, init) {
using BlockDev = AccessHelper<TestDevice>;
using FS = FAT32::FS<BlockDev>;
size_t size = 32*1024*1024;
TestDevice dev(size);
// write non-zero data into the device's memory
for (uint32_t i = 0; i < size; ++i) {dev.buf[i] = 0x55;}
BlockDev bDev(dev);
FS fs(bDev, 0);
// filesystem must not be considered valid, header contains only zeros
ASSERT_FALSE(fs.isValid());
// initialize the filesystem
fs.setup(size, true);
// must be considered valid now
ASSERT_TRUE(fs.isValid());
// size indication must match 32 MiB
ASSERT_EQ(size, fs.getSize());
// data size indication (32 MiB - FATs and reserved sectors)
ASSERT_EQ(size - 81920, fs.getSizeForData());
// currently, one cluster (4096 bytes) is used (for the root dir)
ASSERT_EQ(4096, fs.getSizeUsed());
// there MUST NOT be any valid entry in the root dir, as this is a new FS!
FS::DirIterator di = fs.getRoot();
ASSERT_FALSE(di.nextUsable().isValid());
}
TEST (TestCreat, getOrCreateFile) {
using BlockDev = AccessHelper<TestDevice>;
using FS = FAT32::FS<BlockDev>;
@@ -104,10 +143,22 @@ TEST (TestCreate, getOrCreateFile) {
size_t size = 32*1024*1024;
TestDevice dev(size);
BlockDev bDev(dev);
FS fs(bDev, 0);
fs.setup(size);
FS::File2 f1 = fs.getOrCreateFile("test.txt");
FS fs(bDev, 0);
// filesystem must not be considered valid, header contains only zeros
ASSERT_FALSE(fs.isValid());
// initialize the filesystem
fs.setup(size, true);
// must be considered valid now
ASSERT_TRUE(fs.isValid());
// size indication must match 32 MB
ASSERT_EQ(size, fs.getSize());
FS::File f1 = fs.getOrCreateFile("test.txt");
ASSERT_EQ(0, f1.getSize());
ASSERT_EQ(4096, f1.getAllocatedSize());
ASSERT_EQ("test.txt", f1.getName());
@@ -118,10 +169,76 @@ TEST (TestCreate, getOrCreateFile) {
for (int i = 0; i < 128; ++i) {d1[i] = 1;}
ASSERT_EQ(128, f1.write(128, d1));
FS::File2 f2 = fs.getOrCreateFile("test.txt");
FS::File f2 = fs.getOrCreateFile("test.txt");
ASSERT_EQ(128, f2.getSize());
ASSERT_EQ(128, f2.read(128, d2));
for (uint32_t i = 0; i < 128; ++i) {ASSERT_EQ(d1[i], d2[i]);}
}
template <typename FS, typename DirIter> void dumpStructure(FS& fs, DirIter dirIter) {
while(true) {
auto h = dirIter.nextUsable();
if (!h.isValid()) {return;}
std::cout << h.getName() << std::endl;
if (h.isDot()) {continue;}
if (h.isDotDot()) {continue;}
if (h.isDirectory()) {
dumpStructure(fs, DirIter(fs, h));
}
}
}
TEST (TestCreate, subfolders) {
using BlockDev = AccessHelper<TestDevice>;
using FS = FAT32::FS<BlockDev>;
size_t size = 32*1024*1024;
TestDevice dev(size);
BlockDev bDev(dev);
FS fs(bDev, 0);
fs.setup(size, true);
fs.mkdirs("/test1");
fs.mkdirs("/test1/1");
fs.mkdirs("/test1/2");
fs.mkdirs("/test1/2/3");
fs.mkdirs("/test2/1/2/3");
fs.mkdirs("/test1/a/b");
fs.mkdirs("/test2/x/y");
fs.getOrCreateFile("/hallo.txt");
fs.getOrCreateFile("/test1/x/y/z.txt");
fs.getOrCreateFile("/test3/a/bbbb.txt");
FAT32::DirHandle dh1a = fs.getHandle("/");
FAT32::DirHandle dh1b = fs.getHandle("");
FAT32::DirHandle dh2 = fs.getHandle("/test1");
FAT32::DirHandle dh3 = fs.getHandle("/test1/1");
FS::DirIterator di(fs, 0);
ASSERT_EQ(2, dh1a.getFirstCluster()); ASSERT_TRUE(dh1a.isValid());
ASSERT_EQ(2, dh1b.getFirstCluster()); ASSERT_TRUE(dh1b.isValid());
ASSERT_EQ(3, dh2.getFirstCluster()); ASSERT_TRUE(dh2.isValid());
ASSERT_EQ(4, dh3.getFirstCluster()); ASSERT_TRUE(dh3.isValid());
dev.toFile("/tmp/ram/subdirs.fat32");
// mount -t vfat /tmp/ram/subdirs.fat32 /mnt/fat/ && ls -l /mnt/fat/ && umount /mnt/fat
dumpStructure(fs, fs.getRoot());
}