Data-Recover-Android/app/src/main/java/com/datarecovery/master/utils/ImageDeepDetector.java

package com.datarecovery.master.utils;

import static android.content.Context.POWER_SERVICE;

import android.content.Context;
import android.net.Uri;
import android.os.CancellationSignal;
import android.os.PowerManager;
import android.text.TextUtils;

import com.atmob.common.crypto.CryptoUtils;
import com.datarecovery.master.utils.xfile.XFile;
import com.datarecovery.master.utils.xfile.XFileSearch;
import com.datarecovery.master.utils.xfile.XPathFile;

import org.reactivestreams.Publisher;
import org.reactivestreams.Subscriber;

import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.zip.Adler32;

import atmob.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import atmob.reactivex.rxjava3.annotations.NonNull;
import atmob.reactivex.rxjava3.core.BackpressureStrategy;
import atmob.reactivex.rxjava3.core.Flowable;
import atmob.reactivex.rxjava3.core.FlowableOnSubscribe;
import atmob.reactivex.rxjava3.functions.Function;
import atmob.reactivex.rxjava3.schedulers.Schedulers;

public class ImageDeepDetector {

    private static final int IMAGE_SUFFIX = 1;
    private static final int WECHAT_CACHE = 2;
    private static final int GALLERY_CACHE = 3;
    private static final int JPG_MAGIC = 4;
    private static final int OPPO_GALLERY_CACHE = 5;
    private static final int VIVO_GALLERY_CACHE = 6;
    private static final int XIAOMI_GALLERY_CACHE = 7;
    private static final int MEIZU_GALLERY_CACHE = 8;

    public static Flowable<List<ImageFile>> detect(Context context) {
        return Flowable.create((FlowableOnSubscribe<XFile>) emitter -> {
                    try {
                        CancellationSignal cancellationSignal = XFileSearch.searchExternalStorageAsync(context,
                                new XFileSearch.FileFilter() {
                                    @Override
                                    public boolean acceptFile(XFile file) {
                                        return isAcceptFile(file);
                                    }

                                    @Override
                                    public boolean acceptDirectory(XFile file) {
                                        return isAcceptDirectory(file);
                                    }
                                },
                                new XFileSearch.FileSearchCallback() {
                                    @Override
                                    public void onStart() {
                                    }

                                    @Override
                                    public void onEachFile(XFile file) {
                                        emitter.onNext(file);
                                    }

                                    @Override
                                    public void onFinish() {
                                        emitter.onComplete();
                                    }
                                });
                        emitter.setCancellable(cancellationSignal::cancel);
                    } catch (Exception e) {
                        emitter.onError(e);
                    }
                }, BackpressureStrategy.BUFFER)
                .filter(xFile -> xFile.getTag() != null)
                .flatMap((Function<XFile, Publisher<ImageFile>>) xFile -> {
                    int tag = (int) xFile.getTag();
                    switch (tag) {
                        case JPG_MAGIC:
                        case IMAGE_SUFFIX:
                            return Flowable.just(new ImageFile(xFile));
                        case XIAOMI_GALLERY_CACHE:
                            return Flowable.just(new ImageFile(xFile, ImageFile.CATEGORY_GALLERY));
                        case WECHAT_CACHE:
                            return detectWechatCache(context, xFile);
                        case GALLERY_CACHE:
                            return detectGalleryCache(context, xFile);
                        case OPPO_GALLERY_CACHE:
                            return detectOppoGalleryCache(context, xFile);
                        case VIVO_GALLERY_CACHE:
                            return detectVivoGalleryCache(context, xFile);
                        case MEIZU_GALLERY_CACHE:
                            return detectMeizuGalleryCache(context, xFile);
                        default:
                            return Flowable.empty();
                    }
                })
                .buffer(200, TimeUnit.MILLISECONDS)
                .filter(imageFiles -> imageFiles != null && imageFiles.size() > 0)
                .observeOn(AndroidSchedulers.mainThread())
                .doOnSubscribe(subscription -> {
                    subscription.request(Long.MAX_VALUE);

                    acquireWakeLock(context);
                })
                .doOnCancel(() -> releaseWakeLock(context))
                .doOnTerminate(() -> releaseWakeLock(context));
    }

    private static void releaseWakeLock(Context context) {
        PowerManager powerManager = (PowerManager) context.getSystemService(POWER_SERVICE);
        PowerManager.WakeLock wakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK,
                "ImageDeepDetector::detect");
        if (wakeLock.isHeld()) {
            wakeLock.release();
        }
    }

    private static void acquireWakeLock(Context context) {
        PowerManager powerManager = (PowerManager) context.getSystemService(POWER_SERVICE);
        PowerManager.WakeLock wakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK,
                "ImageDeepDetector::detect");
        wakeLock.acquire(10 * 60 * 1000L /*10 minutes*/);
    }

    private static Flowable<ImageFile> detectMeizuGalleryCache(Context context, XFile xFile) {
        return new GenericImgCollectionDetector(context, xFile, ImageFile.CATEGORY_GALLERY)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectVivoGalleryCache(Context context, XFile xFile) {
        return new GenericImgCollectionDetector(context, xFile, ImageFile.CATEGORY_GALLERY)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectOppoGalleryCache(Context context, XFile xFile) {
        return new GenericImgCollectionDetector(context, xFile, ImageFile.CATEGORY_GALLERY)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectGalleryCache(Context context, XFile xFile) {
        return new GalleryCacheDetector(context, xFile)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectWechatCache(Context context, XFile xFile) {
        return new WechatCacheDetector(context, xFile)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static boolean isAcceptDirectory(XFile file) {
        try {
            String path = file.getPath();
            if (isGalleryCacheDirectory(path)) {
                file.setTag(GALLERY_CACHE);
                return true;
            }
        } catch (Exception ignore) {
        }
        return false;
    }

    private static boolean isAcceptFile(XFile file) {
        try {
            if (file.length() == 0) {
                return false;
            }
        } catch (Exception ignore) {
        }
        try {
            String name = file.getName();
            if (isImageSuffix(name)) {
                file.setTag(IMAGE_SUFFIX);
                return true;
            }
        } catch (Exception ignore) {
        }
        try {
            String path = file.getPath();
            if (isWechatCacheFile(path)) {
                file.setTag(WECHAT_CACHE);
                return true;
            }
            if (isOppoGalleryCacheFile(path)) {
                file.setTag(OPPO_GALLERY_CACHE);
                return true;
            }
            if (isVivoGalleryCacheFile(path)) {
                file.setTag(VIVO_GALLERY_CACHE);
                return true;
            }
            if (isXiaomiGalleryCacheFile(path)) {
                file.setTag(XIAOMI_GALLERY_CACHE);
                return true;
            }
            if (isMeizuGalleryCacheFile(path)) {
                file.setTag(MEIZU_GALLERY_CACHE);
                return true;
            }
        } catch (Exception ignore) {
        }
        if (hasJpgMagic(file)) {
            file.setTag(JPG_MAGIC);
            return true;
        }
        return false;
    }

    private static boolean hasJpgMagic(XFile file) {
        try (InputStream inputStream = file.newInputStream()) {
            if (inputStream.available() < 4) {
                return false;
            }

            byte[] bytes = new byte[2];
            if (inputStream.read(bytes) != 2) {
                return false;
            }
            boolean hasHeaderMagic = bytes[0] == (byte) 0xFF && bytes[1] == (byte) 0xD8;

            if (!hasHeaderMagic) {
                return false;
            }

            long skip = inputStream.available() - 2;
            if (inputStream.skip(skip) != skip) {
                return false;
            }
            if (inputStream.read(bytes) != 2) {
                return false;
            }
            return bytes[0] == (byte) 0xFF && bytes[1] == (byte) 0xD9;
        } catch (Exception ignore) {
        }
        return false;
    }

    private static boolean isGalleryCacheDirectory(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        return path.contains("com.android.gallery3d%2Fcache") ||
                path.contains("com.android.gallery3d/cache");
    }

    private static boolean isWechatCacheFile(String name) {
        if (TextUtils.isEmpty(name)) {
            return false;
        }
        return name.contains("com.tencent.mm%2Fcache%2Fimgcache%2Fcache.data") ||
                name.contains("com.tencent.mm/cache/imgcache/cache.data");
    }

    private static boolean isOppoGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.coloros.gallery3d%2Fcache") &&
                !path.contains("com.coloros.gallery3d/cache")) {
            return false;
        }
        return path.contains("imgcache") || path.contains("screennailcache")
                || path.contains("tilecache");
    }

    private static boolean isVivoGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.vivo.gallery%2Fcache") &&
                !path.contains("com.vivo.gallery/cache")) {
            return false;
        }
        return path.contains("imgcache") || path.contains("trackthumbnail_cache");
    }

    private static boolean isXiaomiGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.miui.gallery%2Ffiles%2Fgallery_disk_cache") &&
                !path.contains("com.miui.gallery/files/gallery_disk_cache")) {
            return false;
        }
        return path.contains("full_size") || path.contains("small_size");
    }

    private static boolean isMeizuGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.meizu.media.gallery%2Fcache") &&
                !path.contains("com.meizu.media.gallery/cache")) {
            return false;
        }
        return path.contains("bestPhotoCache") || path.contains("face_thumbnails")
                || path.contains("uri_thumbnail_cache");
    }

    private static boolean isImageSuffix(String name) {
        if (TextUtils.isEmpty(name)) {
            return false;
        }
        return name.endsWith(".jpg") || name.endsWith(".jpeg") || name.endsWith(".png")
                || name.endsWith(".gif") || name.endsWith(".bmp") || name.endsWith(".webp")
                || name.endsWith(".tiff") || name.endsWith(".psd") || name.endsWith(".svg")
                || name.endsWith(".raw") || name.endsWith(".heif") || name.endsWith(".indd");
    }

    private static boolean bytes2File(byte[] bytes, File file) {
        try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
            fileOutputStream.write(bytes);
            fileOutputStream.flush();
            return true;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return false;
    }

    private static boolean bytes2File(List<Byte> bytes, File file) {
        try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
            for (Byte aByte : bytes) {
                fileOutputStream.write(aByte);
            }
            fileOutputStream.flush();
            return true;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return false;
    }

    private static File getDetectedCacheDir(Context context, String domain) {
        File cacheDir = context.getCacheDir();
        File detectedCacheDir = new File(cacheDir, CryptoUtils.HASH.md5(domain));
        if (!detectedCacheDir.exists()) {
            detectedCacheDir.mkdirs();
        }
        return detectedCacheDir;
    }

    private static void clearDetectedCache(Context context, String domain) {
        File detectedCacheDir = getDetectedCacheDir(context, domain);
        try {
            clearDir(detectedCacheDir);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void clearDir(File dir) {
        if (dir == null || !dir.exists()) {
            return;
        }
        File[] files = dir.listFiles();
        if (files == null || files.length == 0) {
            dir.delete();
            return;
        }
        for (File file : files) {
            if (file.isDirectory()) {
                clearDir(file);
            } else {
                file.delete();
            }
        }
        dir.delete();
    }

    public static class ImageFile {
        public static int CATEGORY_UNKNOWN = -1;
        public static int CATEGORY_OTHER = 0;
        public static int CATEGORY_QQ = 1;
        public static int CATEGORY_WECHAT = 2;
        public static int CATEGORY_GALLERY = 3;
        private final XFile xFile;
        private String name;
        private long size;
        private Uri uri;
        private String path;
        private int category;

        public ImageFile(XFile xFile) {
            this(xFile, CATEGORY_UNKNOWN);
        }

        public ImageFile(XFile xFile, int category) {
            this.category = category;
            this.xFile = xFile;
            try {
                this.name = xFile.getName();
            } catch (Exception ignore) {
            }
            try {
                this.size = xFile.length();
            } catch (Exception ignore) {
            }
            try {
                this.uri = xFile.getUri();
            } catch (Exception ignore) {
            }
            try {
                this.path = xFile.getPath();
            } catch (Exception ignore) {
            }
        }

        public String getName() {
            return name;
        }

        public long getSize() {
            return size;
        }

        public Uri getUri() {
            return uri;
        }

        public InputStream newInputStream() throws Exception {
            return xFile.newInputStream();
        }

        public boolean delete() throws Exception {
            return xFile.delete();
        }

        public int getCategory() {
            if (category != CATEGORY_UNKNOWN) {
                return category;
            }
            if (isGallery()) {
                return CATEGORY_GALLERY;
            } else if (isWechat()) {
                return CATEGORY_WECHAT;
            } else if (isQQ()) {
                return CATEGORY_QQ;
            } else {
                return CATEGORY_OTHER;
            }
        }

        private boolean isQQ() {
            if (TextUtils.isEmpty(path)) {
                return false;
            }
            return path.contains("com.tencent.mobileqq") || path.contains("com.tencent.tim");
        }

        private boolean isWechat() {
            if (!TextUtils.isEmpty(path) && path.contains("com.tencent.mm")) {
                return true;
            }
            return false;
        }

        private boolean isGallery() {
            if (!TextUtils.isEmpty(path) && (
                    path.contains("com.android.gallery3d") ||
                            path.contains("com.coloros.gallery3d") ||
                            path.contains("com.vivo.gallery") ||
                            path.contains("com.miui.gallery") ||
                            path.contains("com.meizu.media.gallery") ||
                            path.contains("com.oppo.gallery3d") ||
                            path.contains("com.android.gallery") ||
                            path.contains("DCIM")
            )) {
                return true;
            }
            return false;
        }
    }

    private static class WechatCacheDetector extends Flowable<ImageFile> {
        private static final String CACHE_DOMAIN = "wechat_cache_detector";
        private final XFile xFile;
        private final Context context;

        public WechatCacheDetector(Context context, XFile xFile) {
            this.context = context;
            this.xFile = xFile;
        }

        @Override
        protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
            long lastModified;
            try {
                lastModified = xFile.lastModified();
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }

            if (checkDetectedCache(context, lastModified, subscriber)) {
                subscriber.onComplete();
                return;
            } else {
                clearDetectedCache(context, CACHE_DOMAIN);
            }

            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            if (!detectedCacheDir.exists()) {
                detectedCacheDir.mkdirs();
            }

            try (InputStream inputStream = xFile.newInputStream()) {
                ArrayList<Byte> imageBytes = new ArrayList<>();
                byte[] buffer = new byte[2048];
                int read;
                while ((read = inputStream.read(buffer)) != -1) {
                    for (int i = 0; i < read; i++) {
                        byte b = buffer[i];
                        imageBytes.add(b);
                        if (imageBytes.size() < 2) {
                            continue;
                        }
                        if (imageBytes.size() == 2) {
                            if (imageBytes.get(0) != (byte) 0xFF || imageBytes.get(1) != (byte) 0xD8) {
                                imageBytes.remove(0);
                            }
                            continue;
                        }
                        if (i == read - 1 && inputStream.available() == 0) {
                            if (imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF && imageBytes.get(imageBytes.size() - 1) == (byte) 0xD9) {
                                File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), ImageFile.CATEGORY_WECHAT));
                                }
                            }
                            imageBytes.clear();
                        } else if (imageBytes.size() >= 6) {
                            if (imageBytes.get(imageBytes.size() - 1) == (byte) 0xD8
                                    && imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF
                                    && imageBytes.get(imageBytes.size() - 3) == (byte) 0xD9
                                    && imageBytes.get(imageBytes.size() - 4) == (byte) 0xFF
                            ) {
                                imageBytes.remove(imageBytes.size() - 1);
                                imageBytes.remove(imageBytes.size() - 1);
                                File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), ImageFile.CATEGORY_WECHAT));
                                }
                                imageBytes.clear();
                                imageBytes.add((byte) 0xFF);
                                imageBytes.add((byte) 0xD8);
                            }
                        }
                    }
                }
                subscriber.onComplete();
            } catch (Exception e) {
                subscriber.onError(e);
            }
        }

        private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
            if (files != null && files.length > 0) {
                for (File file : files) {
                    subscriber.onNext(new ImageFile(new XPathFile(this.context, file), ImageFile.CATEGORY_WECHAT));
                }
                return true;
            }
            return false;
        }
    }

    private static class GalleryCacheDetector extends Flowable<ImageFile> {

        private static final String CACHE_DOMAIN = "gallery_cache_detector";
        private static final int MAGIC_INDEX_FILE = 0xB3273030;
        private static final int MAGIC_DATA_FILE = 0xBD248510;

        // index header offset
        private static final int IH_MAGIC = 0;
        private static final int IH_MAX_ENTRIES = 4;
        private static final int IH_MAX_BYTES = 8;
        private static final int IH_ACTIVE_REGION = 12;
        private static final int IH_ACTIVE_ENTRIES = 16;
        private static final int IH_ACTIVE_BYTES = 20;
        private static final int IH_CHECKSUM = 28;
        private static final int INDEX_HEADER_SIZE = 32;

        private static final int DATA_HEADER_SIZE = 4;

        // blob header offset
        private static final int BH_KEY = 0;
        private static final int BH_CHECKSUM = 8;
        private static final int BH_OFFSET = 12;
        private static final int BH_LENGTH = 16;
        private static final int BLOB_HEADER_SIZE = 20;
        private final XFile galleryCacheDir;
        private final byte[] indexHeader;
        private final byte[] blobHeader;
        private final Adler32 mAdler32 = new Adler32();
        private final Context context;

        private int mMaxEntries;
        private int mMaxBytes;
        private int mActiveRegion;
        private int mActiveBytes;

        private FileChannel mIndexChannel;
        private MappedByteBuffer mIndexBuffer;
        private RandomAccessFile mIndexFile;
        private RandomAccessFile mDataFile0;
        private RandomAccessFile mDataFile1;

        private RandomAccessFile mActiveDataFile;
        private int mActiveHashStart;

        public GalleryCacheDetector(Context context, XFile galleryCacheDir) {
            this.context = context;
            this.galleryCacheDir = galleryCacheDir;
            this.indexHeader = new byte[INDEX_HEADER_SIZE];
            this.blobHeader = new byte[BLOB_HEADER_SIZE];
        }

        @Override
        protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
            XFile[] xFiles;
            try {
                xFiles = galleryCacheDir.listFiles();
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }
            if (xFiles == null || xFiles.length == 0) {
                subscriber.onComplete();
                return;
            }
            XFile indexFile = null;
            XFile dataFile0 = null;
            XFile dataFile1 = null;
            for (XFile xFile : xFiles) {
                try {
                    String name = xFile.getName();
                    if (name.endsWith(".idx")) {
                        indexFile = xFile;
                    } else if (name.endsWith(".0")) {
                        dataFile0 = xFile;
                    } else if (name.endsWith(".1")) {
                        dataFile1 = xFile;
                    }
                } catch (Exception e) {
                    subscriber.onError(e);
                }
            }
            if (indexFile == null || dataFile0 == null || dataFile1 == null) {
                subscriber.onComplete();
                return;
            }
            doDetect(indexFile, dataFile0, dataFile1, subscriber);
        }

        private void doDetect(XFile indexFile, XFile dataFile0, XFile dataFile1, Subscriber<? super ImageFile> subscriber) {
            try {
                long lastModified;
                try {
                    lastModified = indexFile.lastModified();
                } catch (Exception e) {
                    subscriber.onError(e);
                    return;
                }

                if (checkDetectedCache(context, lastModified, subscriber)) {
                    subscriber.onComplete();
                    return;
                } else {
                    clearDetectedCache(context, CACHE_DOMAIN);
                }

                File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
                detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
                if (!detectedCacheDir.exists()) {
                    detectedCacheDir.mkdirs();
                }

                loadIndex(indexFile, dataFile0, dataFile1);
                for (int i = 0; i < mMaxEntries; i++) {
                    int offset = mActiveHashStart + i * 12;
                    long candidateKey = mIndexBuffer.getLong(offset);
                    try {
                        LookupRequest lookupRequest = new LookupRequest(candidateKey);
                        if (!lookup(lookupRequest)) {
                            continue;
                        }
                        byte[] lookup = lookupRequest.buffer;
                        if (lookup == null) {
                            continue;
                        }
                        byte[] cropData = cropLookup(lookup);
                        if (cropData == null) {
                            continue;
                        }
                        File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
                        if (cache.createNewFile() && bytes2File(cropData, cache)) {
                            subscriber.onNext(new ImageFile(new XPathFile(context, cache), ImageFile.CATEGORY_GALLERY));
                        }
                    } catch (Exception ignore) {
                    }
                }
                subscriber.onComplete();
            } catch (Exception e) {
                subscriber.onError(e);
            } finally {
                closeAll();
            }
        }

        private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
            if (files != null && files.length > 0) {
                for (File file : files) {
                    subscriber.onNext(new ImageFile(new XPathFile(this.context, file), ImageFile.CATEGORY_GALLERY));
                }
                return true;
            }
            return false;
        }

        private byte[] cropLookup(byte[] lookup) {
            for (int i = 0; i < lookup.length; i++) {
                if (lookup[i] == (byte) 0xFF && i + 1 < lookup.length && lookup[i + 1] == (byte) 0xD8) {
                    return crop(lookup, i);
                }
            }
            return null;
        }

        private byte[] crop(byte[] lookup, int i) {
            byte[] crop = new byte[lookup.length - i];
            System.arraycopy(lookup, i, crop, 0, crop.length);
            return crop;
        }

        private void loadIndex(XFile indexFile, XFile dataFile0, XFile dataFile1) throws Exception {
            checkFileValid(indexFile, dataFile0, dataFile1);

            try (InputStream idxIs = indexFile.newInputStream();
                 InputStream data0Is = dataFile0.newInputStream();
                 InputStream data1Is = dataFile1.newInputStream()
            ) {
                File indexTemp = createTempFile("index.temp", idxIs);
                mIndexFile = new RandomAccessFile(indexTemp, "rw");

                File data0Temp = createTempFile("data0.temp", data0Is);
                mDataFile0 = new RandomAccessFile(data0Temp, "rw");

                File data1Temp = createTempFile("data1.temp", data1Is);
                mDataFile1 = new RandomAccessFile(data1Temp, "rw");

                // Map index file to memory
                mIndexChannel = mIndexFile.getChannel();
                mIndexBuffer = mIndexChannel.map(FileChannel.MapMode.READ_WRITE,
                        0, mIndexFile.length());
                mIndexBuffer.order(ByteOrder.LITTLE_ENDIAN);

                setActiveVariables();
            }
        }

        private void setActiveVariables() throws Exception {
            mActiveDataFile = (mActiveRegion == 0) ? mDataFile0 : mDataFile1;
            mActiveDataFile.setLength(mActiveBytes);
            mActiveDataFile.seek(mActiveBytes);

            mActiveHashStart = INDEX_HEADER_SIZE;

            if (mActiveRegion != 0) {
                mActiveHashStart += mMaxEntries * 12;
            }
        }

        private void checkFileValid(XFile indexFile, XFile dataFile0, XFile dataFile1) throws Exception {
            byte[] buf = indexHeader;
            try (InputStream inputStream = indexFile.newInputStream()) {
                if (inputStream.read(buf) != INDEX_HEADER_SIZE) {
                    throw new Exception("cannot read header");
                }
                if (readInt(buf, IH_MAGIC) != MAGIC_INDEX_FILE) {
                    throw new Exception("cannot read header magic");
                }
            }

            mMaxEntries = readInt(buf, IH_MAX_ENTRIES);
            mMaxBytes = readInt(buf, IH_MAX_BYTES);
            mActiveRegion = readInt(buf, IH_ACTIVE_REGION);
            int mActiveEntries = readInt(buf, IH_ACTIVE_ENTRIES);
            mActiveBytes = readInt(buf, IH_ACTIVE_BYTES);

            int sum = readInt(buf, IH_CHECKSUM);
            if (checkSum(buf, 0, IH_CHECKSUM) != sum) {
                throw new Exception("header checksum does not match");
            }

            // Sanity check
            if (mMaxEntries <= 0) {
                throw new Exception("invalid max entries");
            }
            if (mMaxBytes <= 0) {
                throw new Exception("invalid max bytes");
            }
            if (mActiveRegion != 0 && mActiveRegion != 1) {
                throw new Exception("invalid active region");
            }
            if (mActiveEntries < 0 || mActiveEntries > mMaxEntries) {
                throw new Exception("invalid active entries");
            }
            if (mActiveBytes < DATA_HEADER_SIZE || mActiveBytes > mMaxBytes) {
                throw new Exception("invalid active bytes");
            }
            if (indexFile.length() != INDEX_HEADER_SIZE + mMaxEntries * 12 * 2L) {
                throw new Exception("invalid index file length");
            }

            // Make sure data file has magic
            byte[] magic = new byte[4];
            try (InputStream data0Is = dataFile0.newInputStream()) {
                if (data0Is.read(magic) != 4) {
                    throw new Exception("cannot read data file magic");
                }
            }
            if (readInt(magic, 0) != MAGIC_DATA_FILE) {
                throw new Exception("invalid data file magic");
            }
            try (InputStream data1Is = dataFile1.newInputStream()) {
                if (data1Is.read(magic) != 4) {
                    throw new Exception("cannot read data file magic");
                }
            }
            if (readInt(magic, 0) != MAGIC_DATA_FILE) {
                throw new Exception("invalid data file magic");
            }
        }

        private File createTempFile(String fileName, InputStream inputStream) throws Exception {
            File tempFile = new File(context.getCacheDir(), fileName);
            if (tempFile.exists()) {
                tempFile.delete();
            }
            if (!tempFile.createNewFile()) {
                throw new Exception("cannot create temp file");
            }
            try (OutputStream outputStream = new FileOutputStream(tempFile)) {
                copyStream(inputStream, outputStream);
            }
            return tempFile;
        }

        public boolean lookup(LookupRequest req) throws IOException {
            if (lookupInternal(req.key, mActiveHashStart)) {
                return getBlob(mActiveDataFile, mFileOffset, req);
            }
            return false;
        }

        private int mFileOffset;

        private boolean lookupInternal(long key, int hashStart) {
            int slot = (int) (key % mMaxEntries);
            if (slot < 0) slot += mMaxEntries;
            int slotBegin = slot;
            while (true) {
                int offset = hashStart + slot * 12;
                long candidateKey = mIndexBuffer.getLong(offset);
                int candidateOffset = mIndexBuffer.getInt(offset + 8);
                if (candidateOffset == 0) {
                    return false;
                } else if (candidateKey == key) {
                    mFileOffset = candidateOffset;
                    return true;
                } else {
                    if (++slot >= mMaxEntries) {
                        slot = 0;
                    }
                    if (slot == slotBegin) {
                        mIndexBuffer.putInt(hashStart + slot * 12 + 8, 0);
                    }
                }
            }
        }

        private boolean getBlob(RandomAccessFile file, int offset,
                                LookupRequest req) throws IOException {
            byte[] header = blobHeader;
            long oldPosition = file.getFilePointer();
            try {
                file.seek(offset);
                if (file.read(header) != BLOB_HEADER_SIZE) {
                    return false;
                }
                long blobKey = readLong(header, BH_KEY);
                if (blobKey == 0) {
                    return false; // This entry has been cleared.
                }
                if (blobKey != req.key) {
                    return false;
                }
                int sum = readInt(header, BH_CHECKSUM);
                int blobOffset = readInt(header, BH_OFFSET);
                if (blobOffset != offset) {
                    return false;
                }
                int length = readInt(header, BH_LENGTH);
                if (length < 0 || length > mMaxBytes - offset - BLOB_HEADER_SIZE) {
                    return false;
                }
                if (req.buffer == null || req.buffer.length < length) {
                    req.buffer = new byte[length];
                }

                byte[] blob = req.buffer;
                req.length = length;

                if (file.read(blob, 0, length) != length) {
                    return false;
                }
                return checkSum(blob, 0, length) == sum;
            } catch (Throwable t) {
                return false;
            } finally {
                file.seek(oldPosition);
            }
        }

        static int readInt(byte[] buf, int offset) {
            return (buf[offset] & 0xff)
                    | ((buf[offset + 1] & 0xff) << 8)
                    | ((buf[offset + 2] & 0xff) << 16)
                    | ((buf[offset + 3] & 0xff) << 24);
        }

        static long readLong(byte[] buf, int offset) {
            long result = buf[offset + 7] & 0xff;
            for (int i = 6; i >= 0; i--) {
                result = (result << 8) | (buf[offset + i] & 0xff);
            }
            return result;
        }

        int checkSum(byte[] data, int offset, int nbytes) {
            mAdler32.reset();
            mAdler32.update(data, offset, nbytes);
            return (int) mAdler32.getValue();
        }

        private void copyStream(InputStream is, OutputStream os) throws Exception {
            byte[] buf = new byte[2048];
            int n;
            while ((n = is.read(buf)) > 0) {
                os.write(buf, 0, n);
            }
            os.flush();
        }

        private void closeAll() {
            closeSilently(mIndexChannel);
            closeSilently(mIndexFile);
            closeSilently(mDataFile0);
            closeSilently(mDataFile1);
        }

        private void closeSilently(Closeable c) {
            if (c == null) return;
            try {
                c.close();
            } catch (Throwable t) {
                // do nothing
            }
        }

        public static class LookupRequest {
            public long key;        // input: the key to find
            public byte[] buffer;   // input/output: the buffer to store the blob
            public int length;      // output: the length of the blob

            public LookupRequest(long key) {
                this.key = key;
            }
        }
    }

    private static class GenericImgCollectionDetector extends Flowable<ImageFile> {

        private final int category;
        private String CACHE_DOMAIN = "generic_img_collection_detector";
        private final Context context;
        private final XFile xFile;

        public GenericImgCollectionDetector(Context context, XFile xFile, int category) {
            this.context = context;
            this.xFile = xFile;
            this.category = category;
        }

        @Override
        protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
            long lastModified;
            try {
                lastModified = xFile.lastModified();
                CACHE_DOMAIN += xFile.getName();
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }

            if (checkDetectedCache(context, lastModified, subscriber)) {
                subscriber.onComplete();
                return;
            } else {
                clearDetectedCache(context, CACHE_DOMAIN);
            }

            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            if (!detectedCacheDir.exists()) {
                detectedCacheDir.mkdirs();
            }

            try (InputStream inputStream = xFile.newInputStream()) {
                ArrayList<Byte> imageBytes = new ArrayList<>();
                byte[] buffer = new byte[2048];
                int read;
                while ((read = inputStream.read(buffer)) != -1) {
                    for (int i = 0; i < read; i++) {
                        byte b = buffer[i];
                        imageBytes.add(b);
                        if (imageBytes.size() < 2) {
                            continue;
                        }
                        if (imageBytes.size() == 2) {
                            if (imageBytes.get(0) != (byte) 0xFF || imageBytes.get(1) != (byte) 0xD8) {
                                imageBytes.remove(0);
                            }
                            continue;
                        }
                        if (i == read - 1 && inputStream.available() == 0) {
                            if (imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF && imageBytes.get(imageBytes.size() - 1) == (byte) 0xD9) {
                                File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), category));
                                }
                            }
                            imageBytes.clear();
                        } else if (imageBytes.size() >= 4) {
                            if (imageBytes.get(imageBytes.size() - 1) == (byte) 0xD9
                                    && imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF
                            ) {
                                File cache = new File(detectedCacheDir, UUID.randomUUID() + ".jpg");
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), category));
                                }
                                imageBytes.clear();
                            }
                        }
                    }
                }
                subscriber.onComplete();
            } catch (Exception e) {
                subscriber.onError(e);
            }
        }

        private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
            if (files != null && files.length > 0) {
                for (File file : files) {
                    subscriber.onNext(new ImageFile(new XPathFile(this.context, file), category));
                }
                return true;
            }
            return false;
        }
    }
}