Data-Recover-Android/app/src/main/java/com/datarecovery/master/utils/ImageDeepDetector.java

package com.datarecovery.master.utils;

import static android.content.Context.POWER_SERVICE;

import android.app.PendingIntent;
import android.content.Context;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.net.Uri;
import android.os.CancellationSignal;
import android.os.Environment;
import android.os.PowerManager;
import android.provider.MediaStore;
import android.text.TextUtils;

import androidx.databinding.BaseObservable;
import androidx.databinding.Bindable;

import com.atmob.common.crypto.CryptoUtils;
import com.atmob.common.runtime.ActivityUtil;
import com.atmob.common.runtime.ContextUtil;
import com.datarecovery.master.BR;
import com.datarecovery.master.utils.xfile.XFile;
import com.datarecovery.master.utils.xfile.XFileSearch;
import com.datarecovery.master.utils.xfile.XPathFile;

import org.reactivestreams.Publisher;
import org.reactivestreams.Subscriber;

import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.zip.Adler32;

import atmob.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import atmob.reactivex.rxjava3.annotations.NonNull;
import atmob.reactivex.rxjava3.core.BackpressureStrategy;
import atmob.reactivex.rxjava3.core.Flowable;
import atmob.reactivex.rxjava3.core.FlowableOnSubscribe;
import atmob.reactivex.rxjava3.functions.Function;
import atmob.reactivex.rxjava3.schedulers.Schedulers;

public class ImageDeepDetector {

    private static final int IMAGE_SUFFIX = 1;
    private static final int WECHAT_CACHE = 2;
    private static final int GALLERY_CACHE = 3;
    private static final int IMG_MAGIC = 4;
    private static final int OPPO_GALLERY_CACHE = 5;
    private static final int VIVO_GALLERY_CACHE = 6;
    private static final int XIAOMI_GALLERY_CACHE = 7;
    private static final int MEIZU_GALLERY_CACHE = 8;
    private static final int HUAWEI_GALLERY_CACHE = 9;

    public static Flowable<List<ImageFile>> detect(Context context) {
        return Flowable.create((FlowableOnSubscribe<XFile>) emitter -> {
                    try {
                        CancellationSignal cancellationSignal = XFileSearch.searchExternalStorageAsync(context,
                                new XFileSearch.FileFilter() {
                                    @Override
                                    public boolean acceptFile(XFile file) {
                                        return isAcceptFile(file);
                                    }

                                    @Override
                                    public boolean acceptDirectory(XFile file) {
                                        return isAcceptDirectory(file);
                                    }

                                    @Override
                                    public boolean filterDirectory(XFile file) {
                                        return isFilterDirectory(file);
                                    }
                                },
                                new XFileSearch.FileSearchCallback() {
                                    @Override
                                    public void onStart() {
                                    }

                                    @Override
                                    public void onEachFile(XFile file) {
                                        emitter.onNext(file);
                                    }

                                    @Override
                                    public void onFinish() {
                                        emitter.onComplete();
                                    }
                                });
                        emitter.setCancellable(cancellationSignal::cancel);
                    } catch (Exception e) {
                        emitter.onError(e);
                    }
                }, BackpressureStrategy.BUFFER)
                .filter(xFile -> xFile.getTag() != null)
                .flatMap((Function<XFile, Publisher<ImageFile>>) xFile -> {
                    int tag = (int) xFile.getTag();
                    switch (tag) {
                        case IMG_MAGIC:
                        case IMAGE_SUFFIX:
                            return Flowable.just(new ImageFile(xFile));
                        case XIAOMI_GALLERY_CACHE:
                            return Flowable.just(new ImageFile(xFile, ImageFile.CATEGORY_GALLERY));
                        case WECHAT_CACHE:
                            return detectWechatCache(context, xFile);
                        case GALLERY_CACHE:
                            return detectGalleryCache(context, xFile);
                        case OPPO_GALLERY_CACHE:
                            return detectOppoGalleryCache(context, xFile);
                        case VIVO_GALLERY_CACHE:
                            return detectVivoGalleryCache(context, xFile);
                        case MEIZU_GALLERY_CACHE:
                            return detectMeizuGalleryCache(context, xFile);
                        case HUAWEI_GALLERY_CACHE:
                            return detectHuaweiGalleryCache(context, xFile);
                        default:
                            return Flowable.empty();
                    }
                })
                .buffer(200, TimeUnit.MILLISECONDS)
                .filter(imageFiles -> imageFiles != null && imageFiles.size() > 0)
                .observeOn(AndroidSchedulers.mainThread())
                .doOnSubscribe(subscription -> {
                    subscription.request(Long.MAX_VALUE);

                    acquireWakeLock(context);
                })
                .doOnCancel(() -> releaseWakeLock(context))
                .doOnTerminate(() -> releaseWakeLock(context));
    }


    private static void releaseWakeLock(Context context) {
        PowerManager powerManager = (PowerManager) context.getSystemService(POWER_SERVICE);
        PowerManager.WakeLock wakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK,
                "ImageDeepDetector::detect");
        if (wakeLock.isHeld()) {
            wakeLock.release();
        }
    }

    private static void acquireWakeLock(Context context) {
        PowerManager powerManager = (PowerManager) context.getSystemService(POWER_SERVICE);
        PowerManager.WakeLock wakeLock = powerManager.newWakeLock(PowerManager.PARTIAL_WAKE_LOCK,
                "ImageDeepDetector::detect");
        wakeLock.acquire(10 * 60 * 1000L /*10 minutes*/);
    }

    private static boolean isFilterDirectory(XFile file) {
        try {
            String path = file.getPath();
            if (TextUtils.isEmpty(path)) {
                return false;
            }
            if (path.endsWith(ContextUtil.getContext().getPackageName())) {
                return true;
            }
            if (path.contains("com.kuaishou.nebula") && (path.endsWith("live_gift_store_icon_directory") ||
                    path.endsWith("magic_finger_resource") || path.endsWith("theme_resource") ||
                    path.endsWith("magic_emoji_resource") || path.endsWith(".material_library_resource") ||
                    path.endsWith("sticker_resource") || path.endsWith("preload%2Ficon%2Fcommon") ||
                    path.endsWith("preload/icon/common") || path.endsWith(".emoji")
            )) {
                return true;
            }
            if ((path.contains("com.tencent.mobileqq") || path.endsWith("com.tencent.tim")) && (path.endsWith("qvideo_newvideo_tips") ||
                    path.endsWith("Gameicon") || path.endsWith("html5") || path.endsWith(".preloaduni") ||
                    path.endsWith(".apollo")) || path.endsWith("editor%2Fresources") || path.endsWith("editor/resources") ||
                    path.endsWith("lottie") || path.endsWith(".vaspoke") || path.endsWith("newpoke") ||
                    path.endsWith("qcircle%2Ffile%2Fdownload") || path.endsWith("qcircle/file/download") ||
                    path.endsWith("qzone%2Fzip_cache") || path.endsWith("qzone/zip_cache") || path.endsWith("poke") ||
                    path.endsWith("DoutuRes")) {
                return true;
            }
            if ((path.contains("com.ss.android.article.video") || path.contains("com.ss.android.ugc.aweme")) && (path.endsWith("liveroom") ||
                    path.endsWith("effect") || path.endsWith("card_3d_res") || path.endsWith("weboffline") || path.endsWith("fantasy_lottie_res")
            )) {
                return true;
            }
            if ((path.contains("com.taobao.taobao") || path.contains("com.tmall.wireless")) && (path.endsWith("AVFSCache") ||
                    path.endsWith("gs_fs"))) {
                return true;
            }
            if ((path.contains("com.baidu.BaiduMap")) && (path.endsWith("sticker"))) {
                return true;
            }
            if ((path.contains("com.eg.android.AlipayGphone")) && (path.endsWith("Sandbox") ||
                    path.endsWith("emojiFiles"))) {
                return true;
            }
            if ((path.contains("air.tv.douyu.android")) && (path.endsWith("skin_download_dir"))) {
                return true;
            }
            if ((path.contains("com.kugou.android")) && (path.endsWith("kugou/lyric") ||
                    path.endsWith("kugou%2Flyric"))) {
                return true;
            }
            if ((path.contains("com.ss.android.article.news") || path.contains("com.ss.android.ugc.aweme"))
                    && (path.endsWith("resources%2Fvariety") || path.endsWith("resources/variety"))) {
                return true;
            }
            if ((path.contains("com.autonavi.minimap") || path.contains("com.amap.android.ams")) && (path.endsWith("sharetrip.taxi") ||
                    path.endsWith("sharetrip%2Ftaxi") || path.endsWith("httpcache"))) {
                return true;
            }
            if (path.contains("tencent/MobileQQ/doodle_template") || path.endsWith("tencent%2FMobileQQ%2Fdoodle_template")) {
                return true;
            }
            if (path.contains(".mob_ad/.material") || path.endsWith("mob_ad%2F.material")) {
                return true;
            }
            if (path.contains("bddownload/common") || path.endsWith("bddownload%2Fcommon")) {
                return true;
            }
            if (path.contains("Pictures/.gs_fs") || path.endsWith("Pictures%2F.gs_fs")) {
                return true;
            }
            if (path.endsWith("files/amap") || path.endsWith("files%2Famap")) {
                return true;
            }
            if (path.endsWith("ksadsdk")) {
                return true;
            }
            if (path.endsWith("files%2Fbddownload%2Fimg_download") || path.endsWith("files/bddownload/img_download")) {
                return true;
            }
            if (path.endsWith("__MACOSX")) {
                return true;
            }
        } catch (Exception ignore) {
        }
        return false;
    }

    private static Flowable<ImageFile> detectHuaweiGalleryCache(Context context, XFile xFile) {
        return new HuaweiGalleryCacheDetector(context, xFile)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectMeizuGalleryCache(Context context, XFile xFile) {
        return new GenericImgCollectionDetector(context, xFile, ImageFile.CATEGORY_GALLERY)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectVivoGalleryCache(Context context, XFile xFile) {
        return new GenericImgCollectionDetector(context, xFile, ImageFile.CATEGORY_GALLERY)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectOppoGalleryCache(Context context, XFile xFile) {
        return new GenericImgCollectionDetector(context, xFile, ImageFile.CATEGORY_GALLERY)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectGalleryCache(Context context, XFile xFile) {
        return new GalleryCacheDetector(context, xFile)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static Flowable<ImageFile> detectWechatCache(Context context, XFile xFile) {
        return new WechatCacheDetector(context, xFile)
                .subscribeOn(Schedulers.io())
                .onErrorComplete();
    }

    private static boolean isAcceptDirectory(XFile file) {
        try {
            String path = file.getPath();
            if (isGalleryCacheDirectory(path)) {
                file.setTag(GALLERY_CACHE);
                return true;
            }
        } catch (Exception ignore) {
        }
        return false;
    }

    private static boolean isAcceptFile(XFile file) {
        try {
            if (file.length() == 0) {
                return false;
            }
        } catch (Exception ignore) {
        }
        try {
            String name = file.getName();
            if (isImageSuffix(name)) {
                file.setTag(IMAGE_SUFFIX);
                return true;
            }
        } catch (Exception ignore) {
        }
        try {
            String path = file.getPath();
            if (isWechatCacheFile(path)) {
                file.setTag(WECHAT_CACHE);
                return true;
            }
            if (isOppoGalleryCacheFile(path)) {
                file.setTag(OPPO_GALLERY_CACHE);
                return true;
            }
            if (isVivoGalleryCacheFile(path)) {
                file.setTag(VIVO_GALLERY_CACHE);
                return true;
            }
            if (isXiaomiGalleryCacheFile(path)) {
                file.setTag(XIAOMI_GALLERY_CACHE);
                return true;
            }
            if (isMeizuGalleryCacheFile(path)) {
                file.setTag(MEIZU_GALLERY_CACHE);
                return true;
            }
            if (isHuaweiGalleryCacheFile(path)) {
                file.setTag(HUAWEI_GALLERY_CACHE);
                return true;
            }
        } catch (Exception ignore) {
        }
        if (hasImgMagic(file)) {
            file.setTag(IMG_MAGIC);
            return true;
        }
        return false;
    }

    private static boolean hasImgMagic(XFile file) {
        try (InputStream inputStream = file.newInputStream()) {
            byte[] bytes = new byte[8];
            if (inputStream.read(bytes) != 8) {
                return false;
            }

            if (bytes[0] == (byte) 0x89 && bytes[1] == (byte) 0x50 && bytes[2] == (byte) 0x4E && bytes[3] == (byte) 0x47
                    && bytes[4] == (byte) 0x0D && bytes[5] == (byte) 0x0A && bytes[6] == (byte) 0x1A && bytes[7] == (byte) 0x0A) {
                // png
                return true;
            }

            boolean hasHeaderMagic = bytes[0] == (byte) 0xFF && bytes[1] == (byte) 0xD8 && bytes[2] == (byte) 0xFF; // jpg header
            if (!hasHeaderMagic) {
                return false;
            }

            long skip = inputStream.available() - 2;
            if (inputStream.skip(skip) != skip) {
                return false;
            }
            if (inputStream.read(bytes) != 2) {
                return false;
            }
            return bytes[0] == (byte) 0xFF && bytes[1] == (byte) 0xD9;
        } catch (Exception ignore) {
        }
        return false;
    }

    private static boolean isGalleryCacheDirectory(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        return path.contains("com.android.gallery3d%2Fcache") ||
                path.contains("com.android.gallery3d/cache");
    }

    private static boolean isWechatCacheFile(String name) {
        if (TextUtils.isEmpty(name)) {
            return false;
        }
        return name.contains("com.tencent.mm%2Fcache%2Fimgcache%2Fcache.data") ||
                name.contains("com.tencent.mm/cache/imgcache/cache.data");
    }

    private static boolean isOppoGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.coloros.gallery3d%2Fcache") &&
                !path.contains("com.coloros.gallery3d/cache")) {
            return false;
        }
        return path.contains("imgcache") || path.contains("screennailcache")
                || path.contains("tilecache");
    }

    private static boolean isVivoGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.vivo.gallery%2Fcache") &&
                !path.contains("com.vivo.gallery/cache")) {
            return false;
        }
        return path.contains("imgcache") || path.contains("trackthumbnail_cache");
    }

    private static boolean isXiaomiGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.miui.gallery%2Ffiles%2Fgallery_disk_cache") &&
                !path.contains("com.miui.gallery/files/gallery_disk_cache")) {
            return false;
        }
        return path.contains("full_size") || path.contains("small_size");
    }

    private static boolean isMeizuGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.meizu.media.gallery%2Fcache") &&
                !path.contains("com.meizu.media.gallery/cache")) {
            return false;
        }
        return path.contains("bestPhotoCache") || path.contains("face_thumbnails")
                || path.contains("uri_thumbnail_cache");
    }

    private static boolean isHuaweiGalleryCacheFile(String path) {
        if (TextUtils.isEmpty(path)) {
            return false;
        }
        if (!path.contains("com.huawei.photos%2Ffiles%2Fthumbdb") &&
                !path.contains("com.huawei.photos/files/thumbdb")) {
            return false;
        }
        return path.endsWith("photoshare.db");
    }

    private static boolean isImageSuffix(String name) {
        if (TextUtils.isEmpty(name)) {
            return false;
        }
        return name.endsWith(".jpg") || name.endsWith(".jpeg") || name.endsWith(".png")
                || name.endsWith(".gif") || name.endsWith(".bmp") || name.endsWith(".webp")
                || name.endsWith(".tiff") || name.endsWith(".psd") || name.endsWith(".svg")
                || name.endsWith(".raw") || name.endsWith(".heif") || name.endsWith(".indd");
    }

    private static boolean bytes2File(byte[] bytes, File file) {
        try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
            fileOutputStream.write(bytes);
            fileOutputStream.flush();
            return true;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return false;
    }

    private static boolean bytes2File(List<Byte> bytes, File file) {
        try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
            for (Byte aByte : bytes) {
                fileOutputStream.write(aByte);
            }
            fileOutputStream.flush();
            return true;
        } catch (Exception e) {
            e.printStackTrace();
        }
        return false;
    }

    private static File getDetectedCacheDir(Context context, String domain) {
        File cacheDir;
        if (Objects.equals(Environment.getExternalStorageState(), Environment.MEDIA_MOUNTED)
                && Environment.getExternalStorageDirectory().canWrite()) {
            cacheDir = context.getExternalCacheDir();
        } else {
            cacheDir = context.getCacheDir();
        }
        File detectedCacheDir = new File(cacheDir, CryptoUtils.HASH.md5(domain));
        if (!detectedCacheDir.exists()) {
            detectedCacheDir.mkdirs();
        }
        return detectedCacheDir;
    }

    private static void clearDetectedCache(Context context, String domain) {
        File detectedCacheDir = getDetectedCacheDir(context, domain);
        try {
            clearDir(detectedCacheDir);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void clearDir(File dir) {
        if (dir == null || !dir.exists()) {
            return;
        }
        File[] files = dir.listFiles();
        if (files == null || files.length == 0) {
            dir.delete();
            return;
        }
        for (File file : files) {
            if (file.isDirectory()) {
                clearDir(file);
            } else {
                file.delete();
            }
        }
        dir.delete();
    }

    public static class ImageFile extends BaseObservable {
        public static int CATEGORY_UNKNOWN = -1;
        public static int CATEGORY_OTHER = 0;
        public static int CATEGORY_QQ = 1;
        public static int CATEGORY_WECHAT = 2;
        public static int CATEGORY_GALLERY = 3;
        private final XFile xFile;
        private String name;
        private long size;
        private String sizeDescribe;
        private Uri uri;
        private String path;
        private int category;
        private String fileType;
        private boolean isCheck;
        private long lastModified;


        public ImageFile(XFile xFile) {
            this(xFile, CATEGORY_UNKNOWN);
        }

        public ImageFile(XFile xFile, int category) {
            this.category = category;
            this.xFile = xFile;
            try {
                this.name = xFile.getName();
            } catch (Exception ignore) {
            }
            try {
                this.size = xFile.length();
            } catch (Exception ignore) {
            }
            try {
                this.uri = xFile.getUri();
            } catch (Exception ignore) {
            }
            try {
                this.path = xFile.getPath();
            } catch (Exception ignore) {
            }
            try {
                this.lastModified = xFile.lastModified();
            } catch (Exception ignore) {
            }
            this.fileType = FileUtil.getImageFileType(name);
            this.sizeDescribe = FileUtil.formatShortBytes(this.size);
        }

        public String getPath() {
            return path;
        }

        public long getLastModified() {
            return lastModified;
        }

        public String getSizeDescribe() {
            return sizeDescribe;
        }

        public String getFileType() {
            return fileType;
        }

        public String getName() {
            return name;
        }

        public long getSize() {
            return size;
        }

        public Uri getUri() {
            return uri;
        }

        @Bindable
        public boolean isCheck() {
            return isCheck;
        }

        public void setCheck(boolean check) {
            isCheck = check;
            notifyPropertyChanged(BR.check);
        }

        public InputStream newInputStream() throws Exception {
            return xFile.newInputStream();
        }

        public boolean delete() throws Exception {
            return xFile.delete();
        }

        public int getCategory() {
            if (category != CATEGORY_UNKNOWN) {
                return category;
            }
            if (isGallery()) {
                return CATEGORY_GALLERY;
            } else if (isWechat()) {
                return CATEGORY_WECHAT;
            } else if (isQQ()) {
                return CATEGORY_QQ;
            } else {
                return CATEGORY_OTHER;
            }
        }

        private boolean isQQ() {
            if (TextUtils.isEmpty(path)) {
                return false;
            }
            return path.contains("com.tencent.mobileqq") || path.contains("com.tencent.tim");
        }

        private boolean isWechat() {
            return !TextUtils.isEmpty(path) && path.contains("com.tencent.mm");
        }

        private boolean isGallery() {
            return !TextUtils.isEmpty(path) && (
                    path.contains("com.android.gallery3d") ||
                            path.contains("com.coloros.gallery3d") ||
                            path.contains("com.vivo.gallery") ||
                            path.contains("com.miui.gallery") ||
                            path.contains("com.meizu.media.gallery") ||
                            path.contains("com.oppo.gallery3d") ||
                            path.contains("com.android.gallery") ||
                            path.contains("com.huawei.photos") ||
                            path.contains("DCIM") ||
                            path.contains("Pictures") ||
                            path.contains(".RecycleBin") ||
                            path.contains(".photoShare")
            );
        }
    }

    private static class WechatCacheDetector extends Flowable<ImageFile> {
        private static final String CACHE_DOMAIN = "wechat_cache_detector";
        private final XFile xFile;
        private final Context context;

        public WechatCacheDetector(Context context, XFile xFile) {
            this.context = context;
            this.xFile = xFile;
        }

        @Override
        protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
            long lastModified;
            try {
                lastModified = xFile.lastModified();
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }

            if (checkDetectedCache(context, lastModified, subscriber)) {
                subscriber.onComplete();
                return;
            } else {
                clearDetectedCache(context, CACHE_DOMAIN);
            }

            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            if (!detectedCacheDir.exists()) {
                detectedCacheDir.mkdirs();
            }

            try (InputStream inputStream = xFile.newInputStream()) {
                ArrayList<Byte> imageBytes = new ArrayList<>();
                byte[] buffer = new byte[2048];
                int read;
                while ((read = inputStream.read(buffer)) != -1) {
                    for (int i = 0; i < read; i++) {
                        byte b = buffer[i];
                        imageBytes.add(b);
                        if (imageBytes.size() < 2) {
                            continue;
                        }
                        if (imageBytes.size() == 2) {
                            if (imageBytes.get(0) != (byte) 0xFF || imageBytes.get(1) != (byte) 0xD8) {
                                imageBytes.remove(0);
                            }
                            continue;
                        }
                        if (i == read - 1 && inputStream.available() == 0) {
                            if (imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF && imageBytes.get(imageBytes.size() - 1) == (byte) 0xD9) {
                                File cache = new File(detectedCacheDir, UUID.randomUUID().toString());
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), ImageFile.CATEGORY_WECHAT));
                                }
                            }
                            imageBytes.clear();
                        } else if (imageBytes.size() >= 6) {
                            if (imageBytes.get(imageBytes.size() - 1) == (byte) 0xD8
                                    && imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF
                                    && imageBytes.get(imageBytes.size() - 3) == (byte) 0xD9
                                    && imageBytes.get(imageBytes.size() - 4) == (byte) 0xFF
                            ) {
                                imageBytes.remove(imageBytes.size() - 1);
                                imageBytes.remove(imageBytes.size() - 1);
                                File cache = new File(detectedCacheDir, UUID.randomUUID().toString());
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), ImageFile.CATEGORY_WECHAT));
                                }
                                imageBytes.clear();
                                imageBytes.add((byte) 0xFF);
                                imageBytes.add((byte) 0xD8);
                            }
                        }
                    }
                }
                subscriber.onComplete();
            } catch (Exception e) {
                subscriber.onError(e);
            }
        }

        private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
            if (files != null && files.length > 0) {
                for (File file : files) {
                    subscriber.onNext(new ImageFile(new XPathFile(this.context, file), ImageFile.CATEGORY_WECHAT));
                }
                return true;
            }
            return false;
        }
    }

    private static class GalleryCacheDetector extends Flowable<ImageFile> {

        private static final String CACHE_DOMAIN = "gallery_cache_detector";
        private static final int MAGIC_INDEX_FILE = 0xB3273030;
        private static final int MAGIC_DATA_FILE = 0xBD248510;

        // index header offset
        private static final int IH_MAGIC = 0;
        private static final int IH_MAX_ENTRIES = 4;
        private static final int IH_MAX_BYTES = 8;
        private static final int IH_ACTIVE_REGION = 12;
        private static final int IH_ACTIVE_ENTRIES = 16;
        private static final int IH_ACTIVE_BYTES = 20;
        private static final int IH_CHECKSUM = 28;
        private static final int INDEX_HEADER_SIZE = 32;

        private static final int DATA_HEADER_SIZE = 4;

        // blob header offset
        private static final int BH_KEY = 0;
        private static final int BH_CHECKSUM = 8;
        private static final int BH_OFFSET = 12;
        private static final int BH_LENGTH = 16;
        private static final int BLOB_HEADER_SIZE = 20;
        private final XFile galleryCacheDir;
        private final byte[] indexHeader;
        private final byte[] blobHeader;
        private final Adler32 mAdler32 = new Adler32();
        private final Context context;

        private int mMaxEntries;
        private int mMaxBytes;
        private int mActiveRegion;
        private int mActiveBytes;

        private FileChannel mIndexChannel;
        private MappedByteBuffer mIndexBuffer;
        private RandomAccessFile mIndexFile;
        private RandomAccessFile mDataFile0;
        private RandomAccessFile mDataFile1;

        private RandomAccessFile mActiveDataFile;
        private int mActiveHashStart;
        private File indexTemp;
        private File data0Temp;
        private File data1Temp;

        public GalleryCacheDetector(Context context, XFile galleryCacheDir) {
            this.context = context;
            this.galleryCacheDir = galleryCacheDir;
            this.indexHeader = new byte[INDEX_HEADER_SIZE];
            this.blobHeader = new byte[BLOB_HEADER_SIZE];
        }

        @Override
        protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
            XFile[] xFiles;
            try {
                xFiles = galleryCacheDir.listFiles();
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }
            if (xFiles == null || xFiles.length == 0) {
                subscriber.onComplete();
                return;
            }
            XFile indexFile = null;
            XFile dataFile0 = null;
            XFile dataFile1 = null;
            for (XFile xFile : xFiles) {
                try {
                    String name = xFile.getName();
                    if (name.endsWith(".idx")) {
                        indexFile = xFile;
                    } else if (name.endsWith(".0")) {
                        dataFile0 = xFile;
                    } else if (name.endsWith(".1")) {
                        dataFile1 = xFile;
                    }
                } catch (Exception e) {
                    subscriber.onError(e);
                }
            }
            if (indexFile == null || dataFile0 == null || dataFile1 == null) {
                subscriber.onComplete();
                return;
            }
            doDetect(indexFile, dataFile0, dataFile1, subscriber);
        }

        private void doDetect(XFile indexFile, XFile dataFile0, XFile dataFile1, Subscriber<? super ImageFile> subscriber) {
            try {
                long lastModified;
                try {
                    lastModified = indexFile.lastModified();
                } catch (Exception e) {
                    subscriber.onError(e);
                    return;
                }

                if (checkDetectedCache(context, lastModified, subscriber)) {
                    subscriber.onComplete();
                    return;
                } else {
                    clearDetectedCache(context, CACHE_DOMAIN);
                }

                File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
                detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
                if (!detectedCacheDir.exists()) {
                    detectedCacheDir.mkdirs();
                }

                loadIndex(indexFile, dataFile0, dataFile1);
                for (int i = 0; i < mMaxEntries; i++) {
                    int offset = mActiveHashStart + i * 12;
                    long candidateKey = mIndexBuffer.getLong(offset);
                    try {
                        LookupRequest lookupRequest = new LookupRequest(candidateKey);
                        if (!lookup(lookupRequest)) {
                            continue;
                        }
                        byte[] lookup = lookupRequest.buffer;
                        if (lookup == null) {
                            continue;
                        }
                        byte[] cropData = cropLookup(lookup);
                        if (cropData == null) {
                            continue;
                        }
                        File cache = new File(detectedCacheDir, UUID.randomUUID().toString());
                        if (cache.createNewFile() && bytes2File(cropData, cache)) {
                            subscriber.onNext(new ImageFile(new XPathFile(context, cache), ImageFile.CATEGORY_GALLERY));
                        }
                    } catch (Exception ignore) {
                    }
                }
                subscriber.onComplete();
            } catch (Exception e) {
                subscriber.onError(e);
            } finally {
                closeAll();
                deleteTempFiles();
            }
        }

        private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
            if (files != null && files.length > 0) {
                for (File file : files) {
                    subscriber.onNext(new ImageFile(new XPathFile(this.context, file), ImageFile.CATEGORY_GALLERY));
                }
                return true;
            }
            return false;
        }

        private byte[] cropLookup(byte[] lookup) {
            for (int i = 0; i < lookup.length; i++) {
                if (lookup[i] == (byte) 0xFF && i + 1 < lookup.length && lookup[i + 1] == (byte) 0xD8) {
                    return crop(lookup, i);
                }
            }
            return null;
        }

        private byte[] crop(byte[] lookup, int i) {
            byte[] crop = new byte[lookup.length - i];
            System.arraycopy(lookup, i, crop, 0, crop.length);
            return crop;
        }

        private void loadIndex(XFile indexFile, XFile dataFile0, XFile dataFile1) throws Exception {
            checkFileValid(indexFile, dataFile0, dataFile1);

            try (InputStream idxIs = indexFile.newInputStream();
                 InputStream data0Is = dataFile0.newInputStream();
                 InputStream data1Is = dataFile1.newInputStream()
            ) {
                indexTemp = createTempFile("index.temp", idxIs);
                mIndexFile = new RandomAccessFile(indexTemp, "rw");

                data0Temp = createTempFile("data0.temp", data0Is);
                mDataFile0 = new RandomAccessFile(data0Temp, "rw");

                data1Temp = createTempFile("data1.temp", data1Is);
                mDataFile1 = new RandomAccessFile(data1Temp, "rw");

                // Map index file to memory
                mIndexChannel = mIndexFile.getChannel();
                mIndexBuffer = mIndexChannel.map(FileChannel.MapMode.READ_WRITE,
                        0, mIndexFile.length());
                mIndexBuffer.order(ByteOrder.LITTLE_ENDIAN);

                setActiveVariables();
            }
        }

        private void setActiveVariables() throws Exception {
            mActiveDataFile = (mActiveRegion == 0) ? mDataFile0 : mDataFile1;
            mActiveDataFile.setLength(mActiveBytes);
            mActiveDataFile.seek(mActiveBytes);

            mActiveHashStart = INDEX_HEADER_SIZE;

            if (mActiveRegion != 0) {
                mActiveHashStart += mMaxEntries * 12;
            }
        }

        private void checkFileValid(XFile indexFile, XFile dataFile0, XFile dataFile1) throws Exception {
            byte[] buf = indexHeader;
            try (InputStream inputStream = indexFile.newInputStream()) {
                if (inputStream.read(buf) != INDEX_HEADER_SIZE) {
                    throw new Exception("cannot read header");
                }
                if (readInt(buf, IH_MAGIC) != MAGIC_INDEX_FILE) {
                    throw new Exception("cannot read header magic");
                }
            }

            mMaxEntries = readInt(buf, IH_MAX_ENTRIES);
            mMaxBytes = readInt(buf, IH_MAX_BYTES);
            mActiveRegion = readInt(buf, IH_ACTIVE_REGION);
            int mActiveEntries = readInt(buf, IH_ACTIVE_ENTRIES);
            mActiveBytes = readInt(buf, IH_ACTIVE_BYTES);

            int sum = readInt(buf, IH_CHECKSUM);
            if (checkSum(buf, 0, IH_CHECKSUM) != sum) {
                throw new Exception("header checksum does not match");
            }

            // Sanity check
            if (mMaxEntries <= 0) {
                throw new Exception("invalid max entries");
            }
            if (mMaxBytes <= 0) {
                throw new Exception("invalid max bytes");
            }
            if (mActiveRegion != 0 && mActiveRegion != 1) {
                throw new Exception("invalid active region");
            }
            if (mActiveEntries < 0 || mActiveEntries > mMaxEntries) {
                throw new Exception("invalid active entries");
            }
            if (mActiveBytes < DATA_HEADER_SIZE || mActiveBytes > mMaxBytes) {
                throw new Exception("invalid active bytes");
            }
            if (indexFile.length() != INDEX_HEADER_SIZE + mMaxEntries * 12 * 2L) {
                throw new Exception("invalid index file length");
            }

            // Make sure data file has magic
            byte[] magic = new byte[4];
            try (InputStream data0Is = dataFile0.newInputStream()) {
                if (data0Is.read(magic) != 4) {
                    throw new Exception("cannot read data file magic");
                }
            }
            if (readInt(magic, 0) != MAGIC_DATA_FILE) {
                throw new Exception("invalid data file magic");
            }
            try (InputStream data1Is = dataFile1.newInputStream()) {
                if (data1Is.read(magic) != 4) {
                    throw new Exception("cannot read data file magic");
                }
            }
            if (readInt(magic, 0) != MAGIC_DATA_FILE) {
                throw new Exception("invalid data file magic");
            }
        }

        private File createTempFile(String fileName, InputStream inputStream) throws Exception {
            File tempFile = new File(context.getCacheDir(), fileName);
            if (tempFile.exists()) {
                tempFile.delete();
            }
            if (!tempFile.createNewFile()) {
                throw new Exception("cannot create temp file");
            }
            try (OutputStream outputStream = new FileOutputStream(tempFile)) {
                copyStream(inputStream, outputStream);
            }
            return tempFile;
        }

        public boolean lookup(LookupRequest req) throws IOException {
            if (lookupInternal(req.key, mActiveHashStart)) {
                return getBlob(mActiveDataFile, mFileOffset, req);
            }
            return false;
        }

        private int mFileOffset;

        private boolean lookupInternal(long key, int hashStart) {
            int slot = (int) (key % mMaxEntries);
            if (slot < 0) slot += mMaxEntries;
            int slotBegin = slot;
            while (true) {
                int offset = hashStart + slot * 12;
                long candidateKey = mIndexBuffer.getLong(offset);
                int candidateOffset = mIndexBuffer.getInt(offset + 8);
                if (candidateOffset == 0) {
                    return false;
                } else if (candidateKey == key) {
                    mFileOffset = candidateOffset;
                    return true;
                } else {
                    if (++slot >= mMaxEntries) {
                        slot = 0;
                    }
                    if (slot == slotBegin) {
                        mIndexBuffer.putInt(hashStart + slot * 12 + 8, 0);
                    }
                }
            }
        }

        private boolean getBlob(RandomAccessFile file, int offset,
                                LookupRequest req) throws IOException {
            byte[] header = blobHeader;
            long oldPosition = file.getFilePointer();
            try {
                file.seek(offset);
                if (file.read(header) != BLOB_HEADER_SIZE) {
                    return false;
                }
                long blobKey = readLong(header, BH_KEY);
                if (blobKey == 0) {
                    return false; // This entry has been cleared.
                }
                if (blobKey != req.key) {
                    return false;
                }
                int sum = readInt(header, BH_CHECKSUM);
                int blobOffset = readInt(header, BH_OFFSET);
                if (blobOffset != offset) {
                    return false;
                }
                int length = readInt(header, BH_LENGTH);
                if (length < 0 || length > mMaxBytes - offset - BLOB_HEADER_SIZE) {
                    return false;
                }
                if (req.buffer == null || req.buffer.length < length) {
                    req.buffer = new byte[length];
                }

                byte[] blob = req.buffer;
                req.length = length;

                if (file.read(blob, 0, length) != length) {
                    return false;
                }
                return checkSum(blob, 0, length) == sum;
            } catch (Throwable t) {
                return false;
            } finally {
                file.seek(oldPosition);
            }
        }

        static int readInt(byte[] buf, int offset) {
            return (buf[offset] & 0xff)
                    | ((buf[offset + 1] & 0xff) << 8)
                    | ((buf[offset + 2] & 0xff) << 16)
                    | ((buf[offset + 3] & 0xff) << 24);
        }

        static long readLong(byte[] buf, int offset) {
            long result = buf[offset + 7] & 0xff;
            for (int i = 6; i >= 0; i--) {
                result = (result << 8) | (buf[offset + i] & 0xff);
            }
            return result;
        }

        int checkSum(byte[] data, int offset, int nbytes) {
            mAdler32.reset();
            mAdler32.update(data, offset, nbytes);
            return (int) mAdler32.getValue();
        }

        private void copyStream(InputStream is, OutputStream os) throws Exception {
            byte[] buf = new byte[2048];
            int n;
            while ((n = is.read(buf)) > 0) {
                os.write(buf, 0, n);
            }
            os.flush();
        }

        private void closeAll() {
            closeSilently(mIndexChannel);
            closeSilently(mIndexFile);
            closeSilently(mDataFile0);
            closeSilently(mDataFile1);
        }

        private void closeSilently(Closeable c) {
            if (c == null) return;
            try {
                c.close();
            } catch (Throwable t) {
                // do nothing
            }
        }

        private void deleteTempFiles() {
            deleteSilently(indexTemp);
            deleteSilently(data0Temp);
            deleteSilently(data1Temp);
        }

        private void deleteSilently(File tempFile) {
            if (tempFile == null) {
                return;
            }
            try {
                if (tempFile.exists()) {
                    tempFile.delete();
                }
            } catch (Throwable t) {
                // do nothing
            }
        }

        public static class LookupRequest {
            public long key;        // input: the key to find
            public byte[] buffer;   // input/output: the buffer to store the blob
            public int length;      // output: the length of the blob

            public LookupRequest(long key) {
                this.key = key;
            }
        }
    }

    private static class GenericImgCollectionDetector extends Flowable<ImageFile> {

        private final int category;
        private String CACHE_DOMAIN = "generic_img_collection_detector";
        private final Context context;
        private final XFile xFile;

        public GenericImgCollectionDetector(Context context, XFile xFile, int category) {
            this.context = context;
            this.xFile = xFile;
            this.category = category;
        }

        @Override
        protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
            long lastModified;
            try {
                lastModified = xFile.lastModified();
                CACHE_DOMAIN += xFile.getName();
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }

            if (checkDetectedCache(context, lastModified, subscriber)) {
                subscriber.onComplete();
                return;
            } else {
                clearDetectedCache(context, CACHE_DOMAIN);
            }

            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            if (!detectedCacheDir.exists()) {
                detectedCacheDir.mkdirs();
            }

            try (InputStream inputStream = xFile.newInputStream()) {
                ArrayList<Byte> imageBytes = new ArrayList<>();
                byte[] buffer = new byte[2048];
                int read;
                while ((read = inputStream.read(buffer)) != -1) {
                    for (int i = 0; i < read; i++) {
                        byte b = buffer[i];
                        imageBytes.add(b);
                        if (imageBytes.size() < 3) {
                            continue;
                        }
                        if (imageBytes.size() == 3) {
                            if (imageBytes.get(0) != (byte) 0xFF || imageBytes.get(1) != (byte) 0xD8 || imageBytes.get(2) != (byte) 0xFF) {
                                imageBytes.remove(0);
                            }
                            continue;
                        }
                        if (i == read - 1 && inputStream.available() == 0) {
                            if (imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF && imageBytes.get(imageBytes.size() - 1) == (byte) 0xD9) {
                                File cache = new File(detectedCacheDir, UUID.randomUUID().toString());
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), category));
                                }
                            }
                            imageBytes.clear();
                        } else if (imageBytes.size() >= 5) {
                            if (imageBytes.get(imageBytes.size() - 1) == (byte) 0xD9
                                    && imageBytes.get(imageBytes.size() - 2) == (byte) 0xFF
                            ) {
                                File cache = new File(detectedCacheDir, UUID.randomUUID().toString());
                                if (cache.createNewFile() && bytes2File(imageBytes, cache)) {
                                    subscriber.onNext(new ImageFile(new XPathFile(context, cache), category));
                                }
                                imageBytes.clear();
                            }
                        }
                    }
                }
                subscriber.onComplete();
            } catch (Exception e) {
                subscriber.onError(e);
            }
        }

        private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
            if (files != null && files.length > 0) {
                for (File file : files) {
                    subscriber.onNext(new ImageFile(new XPathFile(this.context, file), category));
                }
                return true;
            }
            return false;
        }
    }

    private static class HuaweiGalleryCacheDetector extends Flowable<ImageFile> {

        private static final String CACHE_DOMAIN = "huawei_gallery_cache_detector";
        private final Context context;
        private final XFile dbFile;

        public HuaweiGalleryCacheDetector(Context context, XFile dbFile) {
            this.context = context;
            this.dbFile = dbFile;
        }

        @Override
        protected void subscribeActual(@NonNull Subscriber<? super ImageFile> subscriber) {
            long lastModified;
            try {
                lastModified = dbFile.lastModified();
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }

            if (checkDetectedCache(context, lastModified, subscriber)) {
                subscriber.onComplete();
                return;
            } else {
                clearDetectedCache(context, CACHE_DOMAIN);
            }

            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            detectedCacheDir = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            if (!detectedCacheDir.exists()) {
                detectedCacheDir.mkdirs();
            }

            File dbTempFile;
            try {
                dbTempFile = createDbTempFile(detectedCacheDir);
            } catch (Exception e) {
                subscriber.onError(e);
                return;
            }

            try (SQLiteDatabase sqLiteDatabase = SQLiteDatabase.openDatabase(dbTempFile.getPath(), null, SQLiteDatabase.OPEN_READONLY);
                 Cursor cursor = sqLiteDatabase.rawQuery("select * from general_kv", null)
            ) {
                int vIndex = cursor.getColumnIndex("v");
                if (vIndex == -1) {
                    subscriber.onComplete();
                    return;
                }
                while (cursor.moveToNext()) {
                    byte[] data = cursor.getBlob(vIndex);
                    if (data == null || data.length == 0) {
                        continue;
                    }
                    File cache = new File(detectedCacheDir, UUID.randomUUID().toString());
                    if (cache.createNewFile() && bytes2File(data, cache)) {
                        subscriber.onNext(new ImageFile(new XPathFile(context, cache), ImageFile.CATEGORY_GALLERY));
                    }
                }
                subscriber.onComplete();
            } catch (Exception e) {
                subscriber.onError(e);
            } finally {
                if (dbTempFile != null && dbTempFile.exists()) {
                    dbTempFile.delete();
                }
            }
        }

        private File createDbTempFile(File detectedCacheDir) throws Exception {
            File dbTempFile = new File(detectedCacheDir, "huawei_gallery_cache_detector.db");
            if (dbTempFile.exists()) {
                dbTempFile.delete();
            }
            try (InputStream inputStream = dbFile.newInputStream();
                 OutputStream outputStream = new FileOutputStream(dbTempFile)
            ) {
                byte[] buffer = new byte[2048];
                int read;
                while ((read = inputStream.read(buffer)) != -1) {
                    outputStream.write(buffer, 0, read);
                }
                outputStream.flush();
                return dbTempFile;
            }
        }

        private boolean checkDetectedCache(Context context, long lastModified, Subscriber<? super ImageFile> subscriber) {
            File detectedCacheDir = getDetectedCacheDir(context, CACHE_DOMAIN);
            File targetCaches = new File(detectedCacheDir, CryptoUtils.HASH.md5(String.valueOf(lastModified)));
            File[] files = targetCaches.exists() && targetCaches.isDirectory() ? targetCaches.listFiles() : null;
            if (files != null && files.length > 0) {
                for (File file : files) {
                    if (file.getName().endsWith(".db")) {
                        continue;
                    }
                    subscriber.onNext(new ImageFile(new XPathFile(this.context, file), ImageFile.CATEGORY_GALLERY));
                }
                return true;
            }
            return false;
        }
    }
}