|
|
@@ -0,0 +1,820 @@
|
|
|
+package com.datarecovery.master.utils;
|
|
|
+
|
|
|
+import android.content.Context;
|
|
|
+import android.net.Uri;
|
|
|
+import android.os.CancellationSignal;
|
|
|
+import android.text.TextUtils;
|
|
|
+
|
|
|
+import com.atmob.common.crypto.CryptoUtils;
|
|
|
+import com.datarecovery.master.utils.xfile.XFile;
|
|
|
+import com.datarecovery.master.utils.xfile.XFileSearch;
|
|
|
+import com.datarecovery.master.utils.xfile.XPathFile;
|
|
|
+
|
|
|
+import org.reactivestreams.Publisher;
|
|
|
+import org.reactivestreams.Subscriber;
|
|
|
+
|
|
|
+import java.io.Closeable;
|
|
|
+import java.io.File;
|
|
|
+import java.io.FileOutputStream;
|
|
|
+import java.io.IOException;
|
|
|
+import java.io.InputStream;
|
|
|
+import java.io.OutputStream;
|
|
|
+import java.io.RandomAccessFile;
|
|
|
+import java.nio.ByteOrder;
|
|
|
+import java.nio.MappedByteBuffer;
|
|
|
+import java.nio.channels.FileChannel;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.List;
|
|
|
+import java.util.UUID;
|
|
|
+import java.util.zip.Adler32;
|
|
|
+
|
|
|
+import atmob.reactivex.rxjava3.annotations.NonNull;
|
|
|
+import atmob.reactivex.rxjava3.core.BackpressureStrategy;
|
|
|
+import atmob.reactivex.rxjava3.core.Flowable;
|
|
|
+import atmob.reactivex.rxjava3.core.FlowableOnSubscribe;
|
|
|
+import atmob.reactivex.rxjava3.functions.Function;
|
|
|
+import atmob.reactivex.rxjava3.schedulers.Schedulers;
|
|
|
+
|
|
|
+public class ImageDeepDetector {
|
|
|
+
|
|
|
+ private static final int IMAGE_SUFFIX = 1;
|
|
|
+ private static final int WECHAT_CACHE = 2;
|
|
|
+ private static final int GALLERY_CACHE = 3;
|
|
|
+ private static final int JPG_MAGIC = 4;
|
|
|
+
|
|
|
+ public static Flowable<ImageFile> detect(Context context) {
|
|
|
+ return Flowable.create((FlowableOnSubscribe<XFile>) emitter -> {
|
|
|
+ try {
|
|
|
+ CancellationSignal cancellationSignal = XFileSearch.searchExternalStorageAsync(context,
|
|
|
+ new XFileSearch.FileFilter() {
|
|
|
+ @Override
|
|
|
+ public boolean acceptFile(XFile file) {
|
|
|
+ return isAcceptFile(file);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public boolean acceptDirectory(XFile file) {
|
|
|
+ return isAcceptDirectory(file);
|
|
|
+ }
|
|
|
+ },
|
|
|
+ new XFileSearch.FileSearchCallback() {
|
|
|
+ @Override
|
|
|
+ public void onStart() {
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void onEachFile(XFile file) {
|
|
|
+ emitter.onNext(file);
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ public void onFinish() {
|
|
|
+ emitter.onComplete();
|
|
|
+ }
|
|
|
+ });
|
|
|
+ emitter.setCancellable(cancellationSignal::cancel);
|
|
|
+ } catch (Exception e) {
|
|
|
+ emitter.onError(e);
|
|
|
+ }
|
|
|
+ }, BackpressureStrategy.BUFFER)
|
|
|
+ .filter(xFile -> xFile.getTag() != null)
|
|
|
+ .flatMap((Function<XFile, Publisher<ImageFile>>) xFile -> {
|
|
|
+ int tag = (int) xFile.getTag();
|
|
|
+ switch (tag) {
|
|
|
+ case JPG_MAGIC:
|
|
|
+ case IMAGE_SUFFIX:
|
|
|
+ return Flowable.just(new ImageFile(xFile));
|
|
|
+ case WECHAT_CACHE:
|
|
|
+ return detectWechatCache(context, xFile);
|
|
|
+ case GALLERY_CACHE:
|
|
|
+ return detectGalleryCache(context, xFile);
|
|
|
+ default:
|
|
|
+ return Flowable.empty();
|
|
|
+ }
|
|
|
+ });
|
|
|
+ }
|
|
|
+
|
|
|
+ private static Flowable<ImageFile> detectGalleryCache(Context context, XFile xFile) {
|
|
|
+ return new GalleryCacheDetector(context, xFile)
|
|
|
+ .subscribeOn(Schedulers.io())
|
|
|
+ .onErrorComplete();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static Flowable<ImageFile> detectWechatCache(Context context, XFile xFile) {
|
|
|
+ return new WechatCacheDetector(context, xFile)
|
|
|
+ .subscribeOn(Schedulers.io())
|
|
|
+ .onErrorComplete();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean isAcceptDirectory(XFile file) {
|
|
|
+ try {
|
|
|
+ String path = file.getPath();
|
|
|
+ if (isGalleryCacheDirectory(path)) {
|
|
|
+ file.setTag(GALLERY_CACHE);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean isAcceptFile(XFile file) {
|
|
|
+ try {
|
|
|
+ if (file.length() == 0) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ String name = file.getName();
|
|
|
+ if (isImageSuffix(name)) {
|
|
|
+ file.setTag(IMAGE_SUFFIX);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ String path = file.getPath();
|
|
|
+ if (isWechatCacheFile(path)) {
|
|
|
+ file.setTag(WECHAT_CACHE);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ if (hasJpgMagic(file)) {
|
|
|
+ file.setTag(JPG_MAGIC);
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean hasJpgMagic(XFile file) {
|
|
|
+ try (InputStream inputStream = file.newInputStream()) {
|
|
|
+ if (inputStream.available() < 4) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ byte[] bytes = new byte[2];
|
|
|
+ if (inputStream.read(bytes) != 2) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ boolean hasHeaderMagic = bytes[0] == (byte) 0xFF && bytes[1] == (byte) 0xD8;
|
|
|
+
|
|
|
+ if (!hasHeaderMagic) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ long skip = inputStream.available() - 2;
|
|
|
+ if (inputStream.skip(skip) != skip) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ if (inputStream.read(bytes) != 2) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return bytes[0] == (byte) 0xFF && bytes[1] == (byte) 0xD9;
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean isGalleryCacheDirectory(String path) {
|
|
|
+ if (TextUtils.isEmpty(path)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return path.contains("com.android.gallery3d%2Fcache") ||
|
|
|
+ path.contains("com.android.gallery3d/cache");
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean isWechatCacheFile(String name) {
|
|
|
+ if (TextUtils.isEmpty(name)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return name.contains("com.tencent.mm%2Fcache%2Fimgcache%2Fcache.data") ||
|
|
|
+ name.contains("com.tencent.mm/cache/imgcache/cache.data");
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean isImageSuffix(String name) {
|
|
|
+ if (TextUtils.isEmpty(name)) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return name.endsWith(".jpg") || name.endsWith(".jpeg") || name.endsWith(".png")
|
|
|
+ || name.endsWith(".gif") || name.endsWith(".bmp") || name.endsWith(".webp")
|
|
|
+ || name.endsWith(".tiff") || name.endsWith(".psd") || name.endsWith(".svg")
|
|
|
+ || name.endsWith(".raw") || name.endsWith(".heif") || name.endsWith(".indd");
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean bytes2File(byte[] bytes, File file) {
|
|
|
+ try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
|
|
|
+ fileOutputStream.write(bytes);
|
|
|
+ fileOutputStream.flush();
|
|
|
+ return true;
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static boolean bytes2File(List<Byte> bytes, File file) {
|
|
|
+ try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
|
|
|
+ for (Byte aByte : bytes) {
|
|
|
+ fileOutputStream.write(aByte);
|
|
|
+ }
|
|
|
+ fileOutputStream.flush();
|
|
|
+ return true;
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static File getDetectedCacheDir(Context context, String domain) {
|
|
|
+ File cacheDir = context.getCacheDir();
|
|
|
+ File detectedCacheDir = new File(cacheDir, CryptoUtils.HASH.md5(domain));
|
|
|
+ if (!detectedCacheDir.exists()) {
|
|
|
+ detectedCacheDir.mkdirs();
|
|
|
+ }
|
|
|
+ return detectedCacheDir;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void clearDetectedCache(Context context, String domain) {
|
|
|
+ File detectedCacheDir = getDetectedCacheDir(context, domain);
|
|
|
+ try {
|
|
|
+ clearDir(detectedCacheDir);
|
|
|
+ } catch (Exception e) {
|
|
|
+ e.printStackTrace();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static void clearDir(File dir) {
|
|
|
+ if (dir == null || !dir.exists()) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ File[] files = dir.listFiles();
|
|
|
+ if (files == null || files.length == 0) {
|
|
|
+ dir.delete();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ for (File file : files) {
|
|
|
+ if (file.isDirectory()) {
|
|
|
+ clearDir(file);
|
|
|
+ } else {
|
|
|
+ file.delete();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ dir.delete();
|
|
|
+ }
|
|
|
+
|
|
|
+ public static class ImageFile {
|
|
|
+ private final XFile xFile;
|
|
|
+ private String name;
|
|
|
+ private long size;
|
|
|
+ private Uri uri;
|
|
|
+
|
|
|
+ public ImageFile(XFile xFile) {
|
|
|
+ this.xFile = xFile;
|
|
|
+ try {
|
|
|
+ this.name = xFile.getName();
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ this.size = xFile.length();
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ this.uri = xFile.getUri();
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public String getName() {
|
|
|
+ return name;
|
|
|
+ }
|
|
|
+
|
|
|
+ public long getSize() {
|
|
|
+ return size;
|
|
|
+ }
|
|
|
+
|
|
|
+ public Uri getUri() {
|
|
|
+ return uri;
|
|
|
+ }
|
|
|
+
|
|
|
+ public InputStream newInputStream() throws Exception {
|
|
|
+ return xFile.newInputStream();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static class WechatCacheDetector extends Flowable<ImageFile> {
|
|
|
+ private static final String CACHE_DOMAIN = "wechat_cache_detector";
|
|
|
+ private final XFile xFile;
|
|
|
+ private final Context context;
|
|
|
+
|
|
|
+ public WechatCacheDetector(Context context, XFile xFile) {
|
|
|
+ this.context = context;
|
|
|
+ this.xFile = xFile;
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
|
|
|
+ long lastModified;
|
|
|
+ try {
|
|
|
+ lastModified = xFile.lastModified();
|
|
|
+ } catch (Exception e) {
|
|
|
+ subscriber.onError(e);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (checkDetectedCache(context, lastModified, subscriber)) {
|
|
|
+ subscriber.onComplete();
|
|
|
+ return;
|
|
|
+ } else {
|
|
|
+ clearDetectedCache(context, CACHE_DOMAIN);
|
|
|
+ }
|
|
|
+
|
|
|
+ File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
|
|
|
+ detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
|
|
|
+ if (!detectedCacheDir.exists()) {
|
|
|
+ detectedCacheDir.mkdirs();
|
|
|
+ }
|
|
|
+
|
|
|
+ try (InputStream inputStream = xFile.newInputStream()) {
|
|
|
+ ArrayList<Byte> imageBytes = new ArrayList<>();
|
|
|
+ byte[] buffer = new byte[1024];
|
|
|
+ int read;
|
|
|
+ while ((read = inputStream.read(buffer)) != -1) {
|
|
|
+ for (int i = 0; i < read; i++) {
|
|
|
+ byte b = buffer[i];
|
|
|
+ imageBytes.add(b);
|
|
|
+ if (imageBytes.size() < 2) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (imageBytes.size() == 2) {
|
|
|
+ if (imageBytes.get(0) != (byte) 0xFF || imageBytes.get(1) != (byte) 0xD8) {
|
|
|
+ imageBytes.remove(0);
|
|
|
+ }
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ if (i == read - 1 && inputStream.available() == 0) {
|
|
|
+ if (imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF && imageBytes.get(imageBytes.size() - 1) == (byte) 0xD9) {
|
|
|
+ File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
|
|
|
+ if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
|
|
|
+ subscriber.onNext(new ImageFile(new XPathFile(context, cache)));
|
|
|
+ }
|
|
|
+ }
|
|
|
+ imageBytes.clear();
|
|
|
+ } else if (imageBytes.size() >= 6) {
|
|
|
+ if (imageBytes.get(imageBytes.size() - 1) == (byte) 0xD8
|
|
|
+ && imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF
|
|
|
+ && imageBytes.get(imageBytes.size() - 3) == (byte) 0xD9
|
|
|
+ && imageBytes.get(imageBytes.size() - 4) == (byte) 0xFF
|
|
|
+ ) {
|
|
|
+ imageBytes.remove(imageBytes.size() - 1);
|
|
|
+ imageBytes.remove(imageBytes.size() - 1);
|
|
|
+ File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
|
|
|
+ if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
|
|
|
+ subscriber.onNext(new ImageFile(new XPathFile(context, cache)));
|
|
|
+ }
|
|
|
+ imageBytes.clear();
|
|
|
+ imageBytes.add((byte) 0xFF);
|
|
|
+ imageBytes.add((byte) 0xD8);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ subscriber.onComplete();
|
|
|
+ } catch (Exception e) {
|
|
|
+ subscriber.onError(e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
|
|
|
+ File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
|
|
|
+ File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
|
|
|
+ File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
|
|
|
+ if (files != null && files.length > 0) {
|
|
|
+ for (File file : files) {
|
|
|
+ subscriber.onNext(new ImageFile(new XPathFile(this.context, file)));
|
|
|
+ }
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static class GalleryCacheDetector extends Flowable<ImageFile> {
|
|
|
+
|
|
|
+ private static final String CACHE_DOMAIN = "gallery_cache_detector";
|
|
|
+ private static final int MAGIC_INDEX_FILE = 0xB3273030;
|
|
|
+ private static final int MAGIC_DATA_FILE = 0xBD248510;
|
|
|
+
|
|
|
+ // index header offset
|
|
|
+ private static final int IH_MAGIC = 0;
|
|
|
+ private static final int IH_MAX_ENTRIES = 4;
|
|
|
+ private static final int IH_MAX_BYTES = 8;
|
|
|
+ private static final int IH_ACTIVE_REGION = 12;
|
|
|
+ private static final int IH_ACTIVE_ENTRIES = 16;
|
|
|
+ private static final int IH_ACTIVE_BYTES = 20;
|
|
|
+ private static final int IH_CHECKSUM = 28;
|
|
|
+ private static final int INDEX_HEADER_SIZE = 32;
|
|
|
+
|
|
|
+ private static final int DATA_HEADER_SIZE = 4;
|
|
|
+
|
|
|
+ // blob header offset
|
|
|
+ private static final int BH_KEY = 0;
|
|
|
+ private static final int BH_CHECKSUM = 8;
|
|
|
+ private static final int BH_OFFSET = 12;
|
|
|
+ private static final int BH_LENGTH = 16;
|
|
|
+ private static final int BLOB_HEADER_SIZE = 20;
|
|
|
+ private final XFile galleryCacheDir;
|
|
|
+ private final byte[] indexHeader;
|
|
|
+ private final byte[] blobHeader;
|
|
|
+ private final Adler32 mAdler32 = new Adler32();
|
|
|
+ private final Context context;
|
|
|
+
|
|
|
+ private int mMaxEntries;
|
|
|
+ private int mMaxBytes;
|
|
|
+ private int mActiveRegion;
|
|
|
+ private int mActiveBytes;
|
|
|
+
|
|
|
+ private FileChannel mIndexChannel;
|
|
|
+ private MappedByteBuffer mIndexBuffer;
|
|
|
+ private RandomAccessFile mIndexFile;
|
|
|
+ private RandomAccessFile mDataFile0;
|
|
|
+ private RandomAccessFile mDataFile1;
|
|
|
+
|
|
|
+ private RandomAccessFile mActiveDataFile;
|
|
|
+ private int mActiveHashStart;
|
|
|
+
|
|
|
+ public GalleryCacheDetector(Context context, XFile galleryCacheDir) {
|
|
|
+ this.context = context;
|
|
|
+ this.galleryCacheDir = galleryCacheDir;
|
|
|
+ this.indexHeader = new byte[INDEX_HEADER_SIZE];
|
|
|
+ this.blobHeader = new byte[BLOB_HEADER_SIZE];
|
|
|
+ }
|
|
|
+
|
|
|
+ @Override
|
|
|
+ protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
|
|
|
+ XFile[] xFiles;
|
|
|
+ try {
|
|
|
+ xFiles = galleryCacheDir.listFiles();
|
|
|
+ } catch (Exception e) {
|
|
|
+ subscriber.onError(e);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ if (xFiles == null || xFiles.length == 0) {
|
|
|
+ subscriber.onComplete();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ XFile indexFile = null;
|
|
|
+ XFile dataFile0 = null;
|
|
|
+ XFile dataFile1 = null;
|
|
|
+ for (XFile xFile : xFiles) {
|
|
|
+ try {
|
|
|
+ String name = xFile.getName();
|
|
|
+ if (name.endsWith(".idx")) {
|
|
|
+ indexFile = xFile;
|
|
|
+ } else if (name.endsWith(".0")) {
|
|
|
+ dataFile0 = xFile;
|
|
|
+ } else if (name.endsWith(".1")) {
|
|
|
+ dataFile1 = xFile;
|
|
|
+ }
|
|
|
+ } catch (Exception e) {
|
|
|
+ subscriber.onError(e);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (indexFile == null || dataFile0 == null || dataFile1 == null) {
|
|
|
+ subscriber.onComplete();
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ doDetect(indexFile, dataFile0, dataFile1, subscriber);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void doDetect(XFile indexFile, XFile dataFile0, XFile dataFile1, Subscriber<? super ImageFile> subscriber) {
|
|
|
+ try {
|
|
|
+ long lastModified;
|
|
|
+ try {
|
|
|
+ lastModified = indexFile.lastModified();
|
|
|
+ } catch (Exception e) {
|
|
|
+ subscriber.onError(e);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (checkDetectedCache(context, lastModified, subscriber)) {
|
|
|
+ subscriber.onComplete();
|
|
|
+ return;
|
|
|
+ } else {
|
|
|
+ clearDetectedCache(context, CACHE_DOMAIN);
|
|
|
+ }
|
|
|
+
|
|
|
+ File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
|
|
|
+ detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
|
|
|
+ if (!detectedCacheDir.exists()) {
|
|
|
+ detectedCacheDir.mkdirs();
|
|
|
+ }
|
|
|
+
|
|
|
+ loadIndex(indexFile, dataFile0, dataFile1);
|
|
|
+ for (int i = 0; i < mMaxEntries; i++) {
|
|
|
+ int offset = mActiveHashStart + i * 12;
|
|
|
+ long candidateKey = mIndexBuffer.getLong(offset);
|
|
|
+ try {
|
|
|
+ LookupRequest lookupRequest = new LookupRequest(candidateKey);
|
|
|
+ if (!lookup(lookupRequest)) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ byte[] lookup = lookupRequest.buffer;
|
|
|
+ if (lookup == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ byte[] cropData = cropLookup(lookup);
|
|
|
+ if (cropData == null) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
|
|
|
+ if (cache.createNewFile() && bytes2File(cropData, cache)) {
|
|
|
+ subscriber.onNext(new ImageFile(new XPathFile(context, cache)));
|
|
|
+ }
|
|
|
+ } catch (Exception ignore) {
|
|
|
+ }
|
|
|
+ }
|
|
|
+ subscriber.onComplete();
|
|
|
+ } catch (Exception e) {
|
|
|
+ subscriber.onError(e);
|
|
|
+ } finally {
|
|
|
+ closeAll();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
|
|
|
+ File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
|
|
|
+ File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
|
|
|
+ File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
|
|
|
+ if (files != null && files.length > 0) {
|
|
|
+ for (File file : files) {
|
|
|
+ subscriber.onNext(new ImageFile(new XPathFile(this.context, file)));
|
|
|
+ }
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private byte[] cropLookup(byte[] lookup) {
|
|
|
+ for (int i = 0; i < lookup.length; i++) {
|
|
|
+ if (lookup[i] == (byte) 0xFF && i + 1 < lookup.length && lookup[i + 1] == (byte) 0xD8) {
|
|
|
+ return crop(lookup, i);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ private byte[] crop(byte[] lookup, int i) {
|
|
|
+ byte[] crop = new byte[lookup.length - i];
|
|
|
+ System.arraycopy(lookup, i, crop, 0, crop.length);
|
|
|
+ return crop;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void loadIndex(XFile indexFile, XFile dataFile0, XFile dataFile1) throws Exception {
|
|
|
+ checkFileValid(indexFile, dataFile0, dataFile1);
|
|
|
+
|
|
|
+ try (InputStream idxIs = indexFile.newInputStream();
|
|
|
+ InputStream data0Is = dataFile0.newInputStream();
|
|
|
+ InputStream data1Is = dataFile1.newInputStream()
|
|
|
+ ) {
|
|
|
+ File indexTemp = createTempFile("index.temp", idxIs);
|
|
|
+ mIndexFile = new RandomAccessFile(indexTemp, "rw");
|
|
|
+
|
|
|
+ File data0Temp = createTempFile("data0.temp", data0Is);
|
|
|
+ mDataFile0 = new RandomAccessFile(data0Temp, "rw");
|
|
|
+
|
|
|
+ File data1Temp = createTempFile("data1.temp", data1Is);
|
|
|
+ mDataFile1 = new RandomAccessFile(data1Temp, "rw");
|
|
|
+
|
|
|
+ // Map index file to memory
|
|
|
+ mIndexChannel = mIndexFile.getChannel();
|
|
|
+ mIndexBuffer = mIndexChannel.map(FileChannel.MapMode.READ_WRITE,
|
|
|
+ 0, mIndexFile.length());
|
|
|
+ mIndexBuffer.order(ByteOrder.LITTLE_ENDIAN);
|
|
|
+
|
|
|
+ setActiveVariables();
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void setActiveVariables() throws Exception {
|
|
|
+ mActiveDataFile = (mActiveRegion == 0) ? mDataFile0 : mDataFile1;
|
|
|
+ mActiveDataFile.setLength(mActiveBytes);
|
|
|
+ mActiveDataFile.seek(mActiveBytes);
|
|
|
+
|
|
|
+ mActiveHashStart = INDEX_HEADER_SIZE;
|
|
|
+
|
|
|
+ if (mActiveRegion != 0) {
|
|
|
+ mActiveHashStart += mMaxEntries * 12;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void checkFileValid(XFile indexFile, XFile dataFile0, XFile dataFile1) throws Exception {
|
|
|
+ byte[] buf = indexHeader;
|
|
|
+ try (InputStream inputStream = indexFile.newInputStream()) {
|
|
|
+ if (inputStream.read(buf) != INDEX_HEADER_SIZE) {
|
|
|
+ throw new Exception("cannot read header");
|
|
|
+ }
|
|
|
+ if (readInt(buf, IH_MAGIC) != MAGIC_INDEX_FILE) {
|
|
|
+ throw new Exception("cannot read header magic");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ mMaxEntries = readInt(buf, IH_MAX_ENTRIES);
|
|
|
+ mMaxBytes = readInt(buf, IH_MAX_BYTES);
|
|
|
+ mActiveRegion = readInt(buf, IH_ACTIVE_REGION);
|
|
|
+ int mActiveEntries = readInt(buf, IH_ACTIVE_ENTRIES);
|
|
|
+ mActiveBytes = readInt(buf, IH_ACTIVE_BYTES);
|
|
|
+
|
|
|
+ int sum = readInt(buf, IH_CHECKSUM);
|
|
|
+ if (checkSum(buf, 0, IH_CHECKSUM) != sum) {
|
|
|
+ throw new Exception("header checksum does not match");
|
|
|
+ }
|
|
|
+
|
|
|
+ // Sanity check
|
|
|
+ if (mMaxEntries <= 0) {
|
|
|
+ throw new Exception("invalid max entries");
|
|
|
+ }
|
|
|
+ if (mMaxBytes <= 0) {
|
|
|
+ throw new Exception("invalid max bytes");
|
|
|
+ }
|
|
|
+ if (mActiveRegion != 0 && mActiveRegion != 1) {
|
|
|
+ throw new Exception("invalid active region");
|
|
|
+ }
|
|
|
+ if (mActiveEntries < 0 || mActiveEntries > mMaxEntries) {
|
|
|
+ throw new Exception("invalid active entries");
|
|
|
+ }
|
|
|
+ if (mActiveBytes < DATA_HEADER_SIZE || mActiveBytes > mMaxBytes) {
|
|
|
+ throw new Exception("invalid active bytes");
|
|
|
+ }
|
|
|
+ if (indexFile.length() != INDEX_HEADER_SIZE + mMaxEntries * 12 * 2L) {
|
|
|
+ throw new Exception("invalid index file length");
|
|
|
+ }
|
|
|
+
|
|
|
+ // Make sure data file has magic
|
|
|
+ byte[] magic = new byte[4];
|
|
|
+ try (InputStream data0Is = dataFile0.newInputStream()) {
|
|
|
+ if (data0Is.read(magic) != 4) {
|
|
|
+ throw new Exception("cannot read data file magic");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (readInt(magic, 0) != MAGIC_DATA_FILE) {
|
|
|
+ throw new Exception("invalid data file magic");
|
|
|
+ }
|
|
|
+ try (InputStream data1Is = dataFile1.newInputStream()) {
|
|
|
+ if (data1Is.read(magic) != 4) {
|
|
|
+ throw new Exception("cannot read data file magic");
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (readInt(magic, 0) != MAGIC_DATA_FILE) {
|
|
|
+ throw new Exception("invalid data file magic");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private File createTempFile(String fileName, InputStream inputStream) throws Exception {
|
|
|
+ File tempFile = new File(context.getCacheDir(), fileName);
|
|
|
+ if (tempFile.exists()) {
|
|
|
+ tempFile.delete();
|
|
|
+ }
|
|
|
+ if (!tempFile.createNewFile()) {
|
|
|
+ throw new Exception("cannot create temp file");
|
|
|
+ }
|
|
|
+ try (OutputStream outputStream = new FileOutputStream(tempFile)) {
|
|
|
+ copyStream(inputStream, outputStream);
|
|
|
+ }
|
|
|
+ return tempFile;
|
|
|
+ }
|
|
|
+
|
|
|
+ public boolean lookup(LookupRequest req) throws IOException {
|
|
|
+ if (lookupInternal(req.key, mActiveHashStart)) {
|
|
|
+ return getBlob(mActiveDataFile, mFileOffset, req);
|
|
|
+ }
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+
|
|
|
+ private int mFileOffset;
|
|
|
+
|
|
|
+ private boolean lookupInternal(long key, int hashStart) {
|
|
|
+ int slot = (int) (key % mMaxEntries);
|
|
|
+ if (slot < 0) slot += mMaxEntries;
|
|
|
+ int slotBegin = slot;
|
|
|
+ while (true) {
|
|
|
+ int offset = hashStart + slot * 12;
|
|
|
+ long candidateKey = mIndexBuffer.getLong(offset);
|
|
|
+ int candidateOffset = mIndexBuffer.getInt(offset + 8);
|
|
|
+ if (candidateOffset == 0) {
|
|
|
+ return false;
|
|
|
+ } else if (candidateKey == key) {
|
|
|
+ mFileOffset = candidateOffset;
|
|
|
+ return true;
|
|
|
+ } else {
|
|
|
+ if (++slot >= mMaxEntries) {
|
|
|
+ slot = 0;
|
|
|
+ }
|
|
|
+ if (slot == slotBegin) {
|
|
|
+ mIndexBuffer.putInt(hashStart + slot * 12 + 8, 0);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private boolean getBlob(RandomAccessFile file, int offset,
|
|
|
+ LookupRequest req) throws IOException {
|
|
|
+ byte[] header = blobHeader;
|
|
|
+ long oldPosition = file.getFilePointer();
|
|
|
+ try {
|
|
|
+ file.seek(offset);
|
|
|
+ if (file.read(header) != BLOB_HEADER_SIZE) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ long blobKey = readLong(header, BH_KEY);
|
|
|
+ if (blobKey == 0) {
|
|
|
+ return false; // This entry has been cleared.
|
|
|
+ }
|
|
|
+ if (blobKey != req.key) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ int sum = readInt(header, BH_CHECKSUM);
|
|
|
+ int blobOffset = readInt(header, BH_OFFSET);
|
|
|
+ if (blobOffset != offset) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ int length = readInt(header, BH_LENGTH);
|
|
|
+ if (length < 0 || length > mMaxBytes - offset - BLOB_HEADER_SIZE) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ if (req.buffer == null || req.buffer.length < length) {
|
|
|
+ req.buffer = new byte[length];
|
|
|
+ }
|
|
|
+
|
|
|
+ byte[] blob = req.buffer;
|
|
|
+ req.length = length;
|
|
|
+
|
|
|
+ if (file.read(blob, 0, length) != length) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ return checkSum(blob, 0, length) == sum;
|
|
|
+ } catch (Throwable t) {
|
|
|
+ return false;
|
|
|
+ } finally {
|
|
|
+ file.seek(oldPosition);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ static int readInt(byte[] buf, int offset) {
|
|
|
+ return (buf[offset] & 0xff)
|
|
|
+ | ((buf[offset + 1] & 0xff) << 8)
|
|
|
+ | ((buf[offset + 2] & 0xff) << 16)
|
|
|
+ | ((buf[offset + 3] & 0xff) << 24);
|
|
|
+ }
|
|
|
+
|
|
|
+ static long readLong(byte[] buf, int offset) {
|
|
|
+ long result = buf[offset + 7] & 0xff;
|
|
|
+ for (int i = 6; i >= 0; i--) {
|
|
|
+ result = (result << 8) | (buf[offset + i] & 0xff);
|
|
|
+ }
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ int checkSum(byte[] data, int offset, int nbytes) {
|
|
|
+ mAdler32.reset();
|
|
|
+ mAdler32.update(data, offset, nbytes);
|
|
|
+ return (int) mAdler32.getValue();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void copyStream(InputStream is, OutputStream os) throws Exception {
|
|
|
+ byte[] buf = new byte[2048];
|
|
|
+ int n;
|
|
|
+ while ((n = is.read(buf)) > 0) {
|
|
|
+ os.write(buf, 0, n);
|
|
|
+ }
|
|
|
+ os.flush();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void closeAll() {
|
|
|
+ closeSilently(mIndexChannel);
|
|
|
+ closeSilently(mIndexFile);
|
|
|
+ closeSilently(mDataFile0);
|
|
|
+ closeSilently(mDataFile1);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void closeSilently(Closeable c) {
|
|
|
+ if (c == null) return;
|
|
|
+ try {
|
|
|
+ c.close();
|
|
|
+ } catch (Throwable t) {
|
|
|
+ // do nothing
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public static class LookupRequest {
|
|
|
+ public long key; // input: the key to find
|
|
|
+ public byte[] buffer; // input/output: the buffer to store the blob
|
|
|
+ public int length; // output: the length of the blob
|
|
|
+
|
|
|
+ public LookupRequest(long key) {
|
|
|
+ this.key = key;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|