Implement OrderedLogger

Run Logfile-Reordering live on the smartphone itself.

.idea/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/sensorreadout.iml" filepath="$PROJECT_DIR$/sensorreadout.iml" />
+      <module fileurl="file://$PROJECT_DIR$/SensorReadout.iml" filepath="$PROJECT_DIR$/SensorReadout.iml" />
       <module fileurl="file://$PROJECT_DIR$/app/app.iml" filepath="$PROJECT_DIR$/app/app.iml" />
     </modules>
   </component>

.idea/modules.xml.orig

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+<<<<<<< HEAD
+      <module fileurl="file://$PROJECT_DIR$/sensorreadout.iml" filepath="$PROJECT_DIR$/sensorreadout.iml" />
+      <module fileurl="file://$PROJECT_DIR$/app/app.iml" filepath="$PROJECT_DIR$/app/app.iml" />
+=======
+      <module fileurl="file:///mnt/Daten/Dokumente/programme/localization/SensorReadout/SensorReadout.iml" filepath="/mnt/Daten/Dokumente/programme/localization/SensorReadout/SensorReadout.iml" />
+      <module fileurl="file:///mnt/Daten/Dokumente/programme/localization/SensorReadout/app/app.iml" filepath="/mnt/Daten/Dokumente/programme/localization/SensorReadout/app/app.iml" />
+>>>>>>> lindenberg
+    </modules>
+  </component>
+</project>

app/build.gradle.orig

@@ -0,0 +1,38 @@
+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 29
+    buildToolsVersion '29.0.2'
+
+    defaultConfig {
+        applicationId "de.fhws.indoor.sensorreadout"
+        minSdkVersion 21
+<<<<<<< HEAD
+        targetSdkVersion 23
+=======
+        targetSdkVersion 29
+>>>>>>> lindenberg
+        versionCode 1
+        versionName "1.0"
+        multiDexEnabled true
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+}
+
+dependencies {
+    implementation 'com.android.support:design:28.0.0'
+    implementation fileTree(include: ['*.jar'], dir: 'libs')
+    testImplementation 'junit:junit:4.12'
+    implementation 'com.android.support:appcompat-v7:28.0.0'
+    implementation 'com.google.android.gms:play-services:12.0.1'
+    implementation 'com.android.support.constraint:constraint-layout:1.1.3'
+    implementation 'org.apmem.tools:layouts:1.10@aar'
+    //compile 'com.google.android.support:wearable:1.3.0'
+    //compile 'com.google.android.gms:play-services-wearable:8.4.0'
+    //provided 'com.google.android.wearable:wearable:1.0.0'
+}

app/src/main/java/de/fhws/indoor/sensorreadout/MainActivity.java

@@ -39,7 +39,8 @@ import de.fhws.indoor.sensorreadout.sensors.LoggerRAM;
 import de.fhws.indoor.sensorreadout.sensors.PedestrianActivity;
 import de.fhws.indoor.sensorreadout.sensors.PedestrianActivityButton;
 import de.fhws.indoor.sensorreadout.sensors.PhoneSensors;
-import de.fhws.indoor.sensorreadout.sensors.UnorderedLogger;
+import de.fhws.indoor.sensorreadout.sensors.OrderedLogger;
+//import de.fhws.indoor.sensorreadout.sensors.UnorderedLogger;
 import de.fhws.indoor.sensorreadout.sensors.WiFi;
 import de.fhws.indoor.sensorreadout.sensors.WiFiRTT;
 import de.fhws.indoor.sensorreadout.sensors.iBeacon;
@@ -57,7 +58,7 @@ public class MainActivity extends Activity {
     private final ArrayList<mySensor> sensors = new ArrayList<mySensor>();
     //private final Logger logger = new Logger(this);
     //private final LoggerRAM logger = new LoggerRAM(this);
-    private final UnorderedLogger logger = new UnorderedLogger(this);
+    private final OrderedLogger logger = new OrderedLogger(this);
     private Button btnStart;
     private Button btnStop;
     private Button btnGround;
@@ -452,25 +453,25 @@ public class MainActivity extends Activity {
                 @Override public void onData(final SensorType id, final long timestamp, final String csv) {return; }
             });
         }
-//        if(activeSensors.contains("WIFI")) {
-//            // log wifi using sensor number 8
-//            final WiFi wifi = new WiFi(this);
-//            sensors.add(wifi);
-//            wifi.setListener(new mySensor.SensorListener() {
-//                @Override public void onData(final long timestamp, final String csv) { add(SensorType.WIFI, csv, timestamp); }
-//                @Override public void onData(final SensorType id, final long timestamp, final String csv) {return; }
-//            });
-//        }
         if(activeSensors.contains("WIFI")) {
-            // log wifi RTT using sensor number 17
+            // log wifi using sensor number 8
-            final WiFiRTT wifirtt = new WiFiRTT(this);
+            final WiFi wifi = new WiFi(this);
-            sensors.add(wifirtt);
+            sensors.add(wifi);
-            wifirtt.setListener(new mySensor.SensorListener() {
+            wifi.setListener(new mySensor.SensorListener() {
-                @Override public void onData(final long timestamp, final String csv) { add(SensorType.WIFIRTT, csv, timestamp); }
+                @Override public void onData(final long timestamp, final String csv) { add(SensorType.WIFI, csv, timestamp); }
-                @Override public void onData(final SensorType id, final long timestamp, final String csv) { add(SensorType.WIFIRTT, csv, timestamp); }
+                @Override public void onData(final SensorType id, final long timestamp, final String csv) {return; }
             });
-
         }
+//        if(activeSensors.contains("WIFI")) {
+//            // log wifi RTT using sensor number 17
+//            final WiFiRTT wifirtt = new WiFiRTT(this);
+//            sensors.add(wifirtt);
+//            wifirtt.setListener(new mySensor.SensorListener() {
+//                @Override public void onData(final long timestamp, final String csv) { add(SensorType.WIFIRTT, csv, timestamp); }
+//                @Override public void onData(final SensorType id, final long timestamp, final String csv) { add(SensorType.WIFIRTT, csv, timestamp); }
+//            });
+//
+//        }
         if(activeSensors.contains("BLUETOOTH")) {
             // bluetooth permission
             if(ActivityCompat.shouldShowRequestPermissionRationale(this,

app/src/main/java/de/fhws/indoor/sensorreadout/sensors/OrderedLogger.java

@@ -0,0 +1,268 @@
+package de.fhws.indoor.sensorreadout.sensors;
+
+import android.content.Context;
+import android.os.SystemClock;
+import android.support.annotation.NonNull;
+import android.util.Log;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * log sensor data to file
+ * Created by Frank on 25.03.2015.
+ * Re-Written by Markus on 20.06.2019.
+ */
+public final class OrderedLogger {
+
+    private static final long CACHED_LINES = 10000;
+
+    private interface ReorderBufferListener {
+        void onCommit(final List<LogEntry> commitSlice);
+    }
+
+    private static class ReorderBuffer {
+        private long cacheSize = 0;
+        private ArrayList<LogEntry> inputCache;
+        private BlockingQueue<ArrayList<LogEntry>> pendingInputCaches;
+        private ReorderThread reorderThread;
+        private ReorderBufferListener commitListener;
+
+        //statistics
+        private AtomicInteger cachedEntries = new AtomicInteger(0);
+        private AtomicInteger writtenEntries = new AtomicInteger(0);
+        private long sizeTotal = 0;
+
+        //                   #################
+        //                   # ReorderBuffer # -> File
+        // ##############    # ------------- #
+        // # InputCache # -> #               #
+        // ##############    #################
+        //
+        // The InputCache is the fast-access side used by the logger, to append entries.
+        // If the InputCache is full, it is committed to the ReorderThread, and swapped
+        // for a new / empty InputCache.
+        // The ReorderThread sorts the ReorderBuffer, commits the upper half to the writer
+        // and inserts the InputCache into the ReorderBuffer
+
+        public ReorderBuffer(long cacheSize, ReorderBufferListener commitListener) {
+            this.cacheSize = cacheSize;
+            this.commitListener = commitListener;
+        }
+
+        public synchronized void start() {
+            inputCache = new ArrayList<>();
+            inputCache.ensureCapacity((int) cacheSize);
+            pendingInputCaches = new ArrayBlockingQueue<>(5);
+            reorderThread = new ReorderThread();
+            cachedEntries.set(0);
+            writtenEntries.set(0);
+            sizeTotal = 0;
+            reorderThread.start();
+        }
+
+        public synchronized void add(LogEntry entry) {
+            inputCache.add(entry);
+            cachedEntries.incrementAndGet();
+            sizeTotal += entry.csv.length();
+            if(inputCache.size() == cacheSize) {
+                // InputCache full, commit to ReorderThread for processing
+                commitInputCache();
+            }
+        }
+
+        public synchronized void flushAndStop() {
+            if(inputCache.size() > 0) {
+                commitInputCache();
+            }
+            // be sure to commit an empty InputCache to signal the ReorderThread
+            commitInputCache();
+            try {
+                reorderThread.join();
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+        }
+
+        private void commitInputCache() {
+            if(!pendingInputCaches.add(inputCache)) {
+                throw new RuntimeException("InputCaches overrun. ReorderThread can't keep up.");
+            }
+            inputCache = new ArrayList<>();
+            inputCache.ensureCapacity((int) cacheSize);
+        }
+
+        public int getEntriesWritten() { return writtenEntries.get(); }
+        public int getEntriesCached() { return cachedEntries.get(); }
+        public long getSizeTotal() { return sizeTotal; }
+
+        private class ReorderThread extends Thread {
+
+            public ReorderThread() {
+                setName("ReorderThread");
+                setPriority(Thread.MIN_PRIORITY);
+            }
+
+            @Override
+            public void run() {
+                ArrayList<LogEntry> reorderBuffer = new ArrayList<>();
+                reorderBuffer.ensureCapacity((int) cacheSize * 2);
+
+                try {
+                    ArrayList<LogEntry> inputCache = null;
+                    while((inputCache = pendingInputCaches.take()) != null) {
+                        boolean draining = inputCache.size() == 0;
+                        Log.d("OrderedLogger[Thread]", "Received InputCache[size:" + inputCache.size() + " -> draining:" + draining + "]");
+                        if(reorderBuffer.size() > (int) cacheSize || draining) {
+                            // we are (either):
+                            // - in running phase, reorderBuffer is filled and we need to commit
+                            // - in draining phase, commit even if reorderBuffer is not full
+                            Collections.sort(reorderBuffer);
+                            List<LogEntry> commitSlice;
+                            if(draining) {
+                                // we received an empty InputCache -> drain the complete ReorderBuffer into the writer.
+                                commitSlice = reorderBuffer.subList(0, reorderBuffer.size());
+                            } else {
+                                // we are mid-flight, drain cacheSize at max
+                                commitSlice = reorderBuffer.subList(0, Math.min((int) cacheSize, reorderBuffer.size()));
+                            }
+                            Log.d("OrderedLogger[Thread]", "Committing[drain:" + draining + "] Chunk[size:" + commitSlice.size() + "]");
+                            commitListener.onCommit(commitSlice);
+                            writtenEntries.addAndGet(commitSlice.size());
+                            cachedEntries.addAndGet(-commitSlice.size());
+                            commitSlice.clear();
+                        }
+                        reorderBuffer.addAll(inputCache);
+
+                        if(reorderBuffer.size() == 0 && draining) {
+                            Log.d("OrderedLogger[Thread]", "Draining finished");
+                            return; // we are done with draining the buffer
+                        }
+                    }
+                } catch(InterruptedException e) {
+                    e.printStackTrace();
+                }
+            }
+        }
+    }
+
+
+
+
+    public static final long BEGINNING_TS = -1;
+
+    private StringBuilder stringBuilder = new StringBuilder();
+    private File file;
+    private FileOutputStream fos;
+    private Context context;
+
+    private ReorderBuffer reorderBuffer;
+
+    /** timestamp of logging start. all entries are relative to this one */
+    private long startTS = 0;
+
+    public OrderedLogger(Context context) {
+        this.context = context;
+    }
+
+    /** start logging (caching a couple of entries in RAM) */
+    public final void start() {
+
+        // start empty
+        stringBuilder.setLength(0);
+        reorderBuffer = new ReorderBuffer(CACHED_LINES, new ReorderBufferListener() {
+            @Override
+            public void onCommit(List<LogEntry> commitSlice) {
+                for(LogEntry entry : commitSlice) {
+                    try {
+                        fos.write(entry.csv.getBytes());
+                    } catch (IOException e) {
+                        e.printStackTrace();
+                    }
+                }
+            }
+        });
+
+        // starting timestamp
+        startTS = SystemClock.elapsedRealtimeNanos();
+
+        // open the output-file immediately (to get permission errors)
+        // but do NOT yet write anything to the file
+        final DataFolder folder = new DataFolder(context, "sensorOutFiles");
+        file = new File(folder.getFolder(), startTS + ".csv");
+
+        try {
+            fos = new FileOutputStream(file);
+            Log.d("logger", "will write to: " + file.toString());
+        } catch (final Exception e) {
+            throw new MyException("error while opening log-file", e);
+        }
+        reorderBuffer.start();
+    }
+
+    /** stop logging and flush RAM-data to the flash-chip */
+    public final void stop() {
+        reorderBuffer.flushAndStop();
+        close();
+    }
+
+    public File getFile() {
+        return file;
+    }
+
+    public int getCurrentBufferSize() {return reorderBuffer.getEntriesCached();}
+    public int getTotalSize() {return (int)reorderBuffer.getSizeTotal();}
+
+    public int getNumEntries() {return reorderBuffer.getEntriesCached();}
+
+    /** add a new CSV entry for the given sensor number to the internal buffer */
+    public final void addCSV(final SensorType sensorNr, final long timestamp, final String csv) {
+        final long relTS = (timestamp == BEGINNING_TS) ? 0 : (timestamp - startTS);
+        if(relTS >= 0) { // drop pre startTS logs (at the beginning, sensors sometimes deliver old values)
+            stringBuilder.append(relTS);	// relative timestamp (uses less space)
+            stringBuilder.append(';');
+            stringBuilder.append(sensorNr.id());
+            stringBuilder.append(';');
+            stringBuilder.append(csv);
+            stringBuilder.append('\n');
+            LogEntry logEntry = new LogEntry(relTS, stringBuilder.toString());
+            reorderBuffer.add(logEntry);
+            stringBuilder.setLength(0);
+        }
+    }
+
+    private final void close() {
+        try {
+            fos.close();
+        } catch (final Exception e) {
+            throw new MyException("error while wriyting log-file", e);
+        }
+    }
+
+    public final long getStartTS() {
+        return startTS;
+    }
+
+
+    private static class LogEntry implements Comparable<LogEntry> {
+        public long timestamp;
+        public String csv;
+
+        public LogEntry(long timestamp, String csv) {
+            this.timestamp = timestamp;
+            this.csv = csv;
+        }
+
+        @Override
+        public int compareTo(@NonNull LogEntry another) {
+            return (timestamp < another.timestamp) ? (-1) : (+1);
+        }
+    }
+}

build.gradle

@@ -11,7 +11,7 @@ buildscript {
         google()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.4.2'
+        classpath 'com.android.tools.build:gradle:3.5.0'
 
         // NOTE: Do not place your application dependencies here; they belong
         // in the individual module build.gradle files

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Sun Aug 18 15:19:02 CEST 2019
+#Tue Apr 07 13:33:43 CEST 2020
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.4.1-all.zip