You break it, you fix it!

The GUI is now working again, with scanning built-in.

12 files changed, 553 insertions, 30 deletions

diff --git a/.gitignore b/.gitignore
index cfb2ba4..790e084 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ util_internal/
 mingui/build/
 
 CMakeLists.txt.user
+*.swp
diff --git a/qdeduper/CMakeLists.txt b/qdeduper/CMakeLists.txt
index f862c16..270cd45 100644
--- a/qdeduper/CMakeLists.txt
+++ b/qdeduper/CMakeLists.txt
@@ -1,6 +1,6 @@
 set(CMAKE_INCLUDE_CURRENT_DIR ON)
 
-find_package(Qt5 REQUIRED COMPONENTS Widgets)
+find_package(Qt5 REQUIRED COMPONENTS Widgets Concurrent)
 
 set(CMAKE_AUTOMOC ON)
 set(CMAKE_AUTORCC ON)
@@ -10,9 +10,13 @@ add_executable(qdeduper
     main.cpp
     mingui.cpp
     imageitem.cpp
+    sigdb_qt.cpp
+    filescanner.cpp
+    pathchooser.cpp
 )
 
 target_link_libraries(qdeduper
     xsig
     Qt5::Widgets
+    Qt5::Concurrent
 )
diff --git a/qdeduper/filescanner.cpp b/qdeduper/filescanner.cpp
new file mode 100644
index 0000000..e7e45fc
--- /dev/null
+++ b/qdeduper/filescanner.cpp
@@ -0,0 +1,75 @@
+#include "filescanner.hpp"
+
+#include <cstring>
+#include <algorithm>
+#include <fstream>
+
+using std::size_t;
+
+FileScanner::FileScanner() : QObject(nullptr), maxmnlen(0)
+{
+
+}
+
+void FileScanner::add_magic_number(const std::string &m)
+{
+    mn.push_back(m);
+    if (m.length() > maxmnlen)
+        maxmnlen = m.length();
+}
+
+void FileScanner::add_path(const fs::path &p, bool recurse)
+{
+    paths.emplace_back(p, recurse);
+}
+
+template <class T>
+void dirit_foreach(T iter, std::function<void(const fs::directory_entry& p)> f)
+{
+    std::for_each(fs::begin(iter), fs::end(iter), f);
+}
+
+void FileScanner::scan()
+{
+    size_t fcnt = 0;
+    auto opt = std::filesystem::directory_options::skip_permission_denied;
+    auto count_files = [&fcnt](const fs::directory_entry& e){
+        if (e.is_regular_file()) ++fcnt;
+    };
+    auto scan_file = [&fcnt, this](const fs::directory_entry &e) {
+        if (!e.is_regular_file()) return;
+        std::fstream fst(e.path(), std::ios::binary | std::ios::in);
+        std::string buf(maxmnlen, '\0');
+        fst.read(buf.data(), maxmnlen);
+        buf.resize(fst.gcount());
+        for (auto &magic : mn)
+            if (!memcmp(magic.data(), buf.data(), magic.length()))
+            {
+                ret.push_back(e.path());
+                break;
+            }
+        Q_EMIT file_scanned(e.path(), ++fcnt);
+    };
+    auto for_all_paths = [opt, this](std::function<void(const fs::directory_entry&)> f) {
+        for (auto &pe : paths)
+        {
+            fs::path p;
+            bool recurse;
+            std::tie(p, recurse) = pe;
+            if (recurse)
+                dirit_foreach(fs::recursive_directory_iterator(p, opt), f);
+            else
+                dirit_foreach(fs::directory_iterator(p, opt), f);
+        }
+    };
+    for_all_paths(count_files);
+    Q_EMIT scan_done_prep(fcnt);
+
+    fcnt = 0;
+    for_all_paths(scan_file);
+}
+
+std::vector<fs::path> FileScanner::file_list()
+{
+    return ret;
+}
diff --git a/qdeduper/filescanner.hpp b/qdeduper/filescanner.hpp
new file mode 100644
index 0000000..5d927a4
--- /dev/null
+++ b/qdeduper/filescanner.hpp
@@ -0,0 +1,30 @@
+#ifndef FILESCANNER_HPP
+#define FILESCANNER_HPP
+
+#include <filesystem>
+#include <string>
+#include <utility>
+#include <vector>
+#include <QObject>
+
+namespace fs = std::filesystem;
+
+class FileScanner : public QObject
+{
+    Q_OBJECT
+    std::vector<std::string> mn;
+    std::vector<std::pair<fs::path, bool>> paths;
+    std::vector<fs::path> ret;
+    std::size_t maxmnlen;
+public:
+    FileScanner();
+    void add_magic_number(const std::string &m);
+    void add_path(const fs::path &p, bool recurse = false);
+    void scan();
+    std::vector<fs::path> file_list();
+Q_SIGNALS:
+    void scan_done_prep(std::size_t nfiles);
+    void file_scanned(const fs::path &p, std::size_t n);
+};
+
+#endif
diff --git a/qdeduper/mingui.cpp b/qdeduper/mingui.cpp
index 317076c..e936a36 100644
--- a/qdeduper/mingui.cpp
+++ b/qdeduper/mingui.cpp
@@ -1,11 +1,18 @@
 #include "mingui.hpp"
 #include "imageitem.hpp"
+#include "filescanner.hpp"
+#include "pathchooser.hpp"
+#include "sigdb_qt.hpp"
 
 #include <cstdio>
+#include <chrono>
 #include <cwchar>
+#include <qnamespace.h>
 #include <type_traits>
 
 #include <QDebug>
+#include <QtConcurrent>
+#include <QFutureWatcher>
 #include <QCloseEvent>
 #include <QMouseEvent>
 #include <QScrollBar>
@@ -16,6 +23,7 @@
 #include <QString>
 #include <QScrollArea>
 #include <QListView>
+#include <QProgressDialog>
 #include <QStandardItemModel>
 #include <QLabel>
 #include <QHBoxLayout>
@@ -83,6 +91,12 @@ DeduperMainWindow::DeduperMainWindow()
     lw->setHorizontalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel);
     lw->setVerticalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel);
     lw->setMinimumWidth(240);
+    pd = new QProgressDialog(this);
+    pd->setModal(true);
+    pd->setMinimumDuration(0);
+    pd->setAutoReset(false);
+    pd->setAutoClose(false);
+    pd->close();
 
     for (size_t i = 0; i < keys.size(); ++i)
     {
@@ -105,29 +119,10 @@ DeduperMainWindow::DeduperMainWindow()
     mnone->setShortcut(QKeySequence(Qt::Key::Key_C));
     QObject::connect(mnone, &QAction::triggered, [this]{this->mark_none();});
     this->addAction(mnone);
-    QAction *nxt = new QAction();
-    nxt->setShortcut(QKeySequence(Qt::Key::Key_M));
-    QObject::connect(nxt, &QAction::triggered, [this]{Q_EMIT this->next();});
-    this->addAction(nxt);
-    QAction *prv = new QAction();
-    prv->setShortcut(QKeySequence(Qt::Key::Key_Z));
-    QObject::connect(prv, &QAction::triggered, [this]{Q_EMIT this->prev();});
-    this->addAction(prv);
     QAction *load = new QAction();
     load->setShortcut(QKeySequence(Qt::Key::Key_N));
     QObject::connect(load, &QAction::triggered, [this]{Q_EMIT this->load_list();});
     this->addAction(load);
-    QAction *skip = new QAction();
-    skip->setShortcut(QKeySequence(Qt::Key::Key_B));
-    QObject::connect(skip, &QAction::triggered, [this]{
-        bool ok = false;
-        int g = QInputDialog::getInt(this, "Skip to group",
-                                     QString("Group # (1-%1)").arg(ngroups),
-                                     curgroup + 1,
-                                     1, ngroups, 1, &ok);
-        if (ok) Q_EMIT switch_group((size_t) g - 1);
-    });
-    this->addAction(skip);
     QAction *save = new QAction();
     save->setShortcut(QKeySequence(Qt::Modifier::SHIFT | Qt::Key::Key_Return));
     QObject::connect(save, &QAction::triggered, [this]{Q_EMIT this->save_list();});
@@ -173,7 +168,9 @@ void DeduperMainWindow::setup_menu()
     QMenu *mark = this->menuBar()->addMenu("Marks");
     QMenu *help = this->menuBar()->addMenu("Help");
 
-    file->addAction("Create Database...");
+    QAction *create_db = file->addAction("Create Database...");
+    QObject::connect(create_db, &QAction::triggered, this, &DeduperMainWindow::create_new);
+    menuact["create_db"] = create_db;
     file->addAction("Load Database...");
     file->addAction("Save Database...");
     file->addSeparator();
@@ -182,8 +179,38 @@ void DeduperMainWindow::setup_menu()
     file->addAction("Preferences...");
     file->addAction("Exit");
 
-    view->addAction("Next Group");
-    view->addAction("Previous Group");
+    QAction *nxtgrp = view->addAction("Next Group");
+    menuact["next_group"] = nxtgrp;
+    nxtgrp->setShortcut(QKeySequence(Qt::Key::Key_M));
+    QObject::connect(nxtgrp, &QAction::triggered, [this] {
+        if (this->sdb && curgroup + 1 < this->sdb->num_groups())
+            this->show_group(++curgroup);
+    });
+    this->addAction(nxtgrp);
+
+    QAction *prvgrp = view->addAction("Previous Group");
+    menuact["prev_group"] = prvgrp;
+    prvgrp->setShortcut(QKeySequence(Qt::Key::Key_Z));
+    QObject::connect(prvgrp, &QAction::triggered, [this] {
+        if (this->sdb && curgroup > 1)
+            this->show_group(--curgroup);
+    });
+    this->addAction(prvgrp);
+
+    QAction *skip = view->addAction("Skip to Group...");
+    menuact["skip_group"] = skip;
+    skip->setShortcut(QKeySequence(Qt::Key::Key_B));
+    QObject::connect(skip, &QAction::triggered, [this] {
+        if (!this->sdb) return;
+        bool ok = false;
+        int g = QInputDialog::getInt(this, "Skip to group",
+                                     QString("Group # (1-%1)").arg(sdb->num_groups()),
+                                     curgroup + 1,
+                                     1, sdb->num_groups(), 1, &ok);
+        if (ok) this->show_group((size_t) g - 1);
+    });
+    this->addAction(skip);
+
     view->addSeparator();
     QMenu *sort = view->addMenu("Sort by");
     sort->addAction("File size");
@@ -198,16 +225,21 @@ void DeduperMainWindow::setup_menu()
     help->addAction("View Documentation");
     help->addAction("About");
 }
+void DeduperMainWindow::update_actions()
+{
+    if (!sdb)
+    {
+        menuact["next_group"]->setEnabled(false);
+        menuact["prev_group"]->setEnabled(false);
+        return;
+    }
+}
 
 void DeduperMainWindow::show_images(const std::vector<fs::path> &fns)
 {
     current_set = fns;
     marks.clear();
     im->clear();
-    int max_height = (this->screen()->size().height() / fns.size() * 0.8 - 24) * this->screen()->devicePixelRatio();
-    int max_width = this->screen()->size().width() * 0.8 * this->screen()->devicePixelRatio();
-    if (max_height < 64) max_height = 64;
-    if (max_width < 64) max_width = 64;
     fs::path::string_type common_pfx = common_prefix(fns);
     size_t idx = 0;
     if (fns.size() > keys.size() && !nohotkeywarn)
@@ -245,7 +277,6 @@ void DeduperMainWindow::update_distances(const std::map<std::pair<size_t, size_t
 void DeduperMainWindow::update_viewstatus(std::size_t cur, std::size_t size)
 {
     permamsg->setText(QString("Viewing group %1 of %2").arg(cur + 1).arg(size));
-    ngroups = size;
     curgroup = cur;
 }
 
@@ -291,6 +322,83 @@ void DeduperMainWindow::load_list()
     mark_view_update();
 }
 
+void DeduperMainWindow::create_new()
+{
+    PathChooser *pc = new PathChooser(this);
+    pc->setModal(true);
+    if (pc->exec() == QDialog::DialogCode::Accepted)
+        this->scan_dirs(pc->get_dirs());
+    pc->deleteLater();
+}
+
+void DeduperMainWindow::scan_dirs(std::vector<std::pair<fs::path, bool>> paths)
+{
+    this->pd->open();
+    this->pd->setLabelText("Preparing for database creation...");
+    this->pd->setMinimum(0);
+    this->pd->setMaximum(0);
+    auto f = QtConcurrent::run([this, paths] {
+        FileScanner *fs = new FileScanner();
+        std::for_each(paths.begin(), paths.end(), [fs](auto p){fs->add_path(p.first, p.second);});
+        fs->add_magic_number("\x89PNG\r\n");
+        fs->add_magic_number("\xff\xd8\xff");
+        QObject::connect(fs, &FileScanner::scan_done_prep, [this](auto n) {
+            this->pd->setMaximum(n - 1);
+        });
+        QObject::connect(fs, &FileScanner::file_scanned, [this](const fs::path &p, size_t c) {
+            static auto lt = std::chrono::steady_clock::now();
+            using namespace std::literals;
+            if (std::chrono::steady_clock::now() - lt > 100ms)
+            {
+                lt = std::chrono::steady_clock::now();
+                this->pd->setLabelText(QString("Looking for files to scan: %1").arg(fsstr_to_qstring(p)));
+                this->pd->setValue(c);
+            }
+        });
+        fs->scan();
+        this->pd->setMaximum(fs->file_list().size() - 1);
+        this->pd->setLabelText("Scanning...");
+        this->sdb = new SignatureDB();
+        QObject::connect(this->sdb, &SignatureDB::image_scanned, this, [this](size_t n) {
+            static auto lt = std::chrono::steady_clock::now();
+            using namespace std::literals;
+            if (std::chrono::steady_clock::now() - lt > 100ms)
+            {
+                lt = std::chrono::steady_clock::now();
+                this->pd->setLabelText(QString("Scanning %1 / %2").arg(n + 1).arg(this->pd->maximum() + 1));
+                this->pd->setValue(n);
+            }
+            if (!~n)
+            {
+                this->pd->setMaximum(0);
+                this->pd->setLabelText("Finalizing...");
+            }
+        }, Qt::ConnectionType::QueuedConnection);
+        this->sdb->scan_files(fs->file_list(), 8);
+        delete fs;
+    });
+    QFutureWatcher<void> *fw = new QFutureWatcher<void>(this);
+    fw->setFuture(f);
+    QObject::connect(fw, &QFutureWatcher<void>::finished, this, [this] {
+        this->pd->reset();
+        this->pd->close();
+        this->curgroup = 0;
+        this->show_group(this->curgroup);
+    }, Qt::ConnectionType::QueuedConnection);
+}
+
+void DeduperMainWindow::show_group(size_t gid)
+{
+    if (!sdb || gid >= sdb->num_groups())
+        return;
+    auto g = sdb->get_group(gid);
+    current_set.clear();
+    std::for_each(g.begin(), g.end(), [this](auto id){current_set.push_back(sdb->get_image_path(id));});
+    this->show_images(current_set);
+    this->update_distances(sdb->group_distances(gid));
+    this->update_viewstatus(gid, sdb->num_groups());
+}
+
 void DeduperMainWindow::mark_toggle(size_t x)
 {
     if (x < marks.size())
diff --git a/qdeduper/mingui.hpp b/qdeduper/mingui.hpp
index b8d13ef..3515047 100644
--- a/qdeduper/mingui.hpp
+++ b/qdeduper/mingui.hpp
@@ -2,6 +2,7 @@
 #define MINGUI_HPP
 
 #include <filesystem>
+#include <map>
 #include <vector>
 #include <string>
 #include <unordered_set>
@@ -9,12 +10,15 @@
 #include <QMainWindow>
 #include <QList>
 
+#include "sigdb_qt.hpp"
+
 class QHBoxLayout;
 class QLabel;
 class QStatusBar;
 class QScrollArea;
 class QTextEdit;
 class QListView;
+class QProgressDialog;
 class QSplitter;
 class QStandardItemModel;
 class ImageItemDelegate;
@@ -30,10 +34,14 @@ private:
     QLabel *permamsg;
     QStatusBar *sb;
     QListView *lw;
+    std::map<std::string, QAction*> menuact;
     QList<QAction*> selhk;
     QStandardItemModel *im = nullptr;
     ImageItemDelegate *id = nullptr;
-    std::size_t ngroups, curgroup;
+    QProgressDialog *pd = nullptr;
+    SignatureDB *sdb = nullptr;
+
+    std::size_t curgroup;
     bool nohotkeywarn;
     void mark_toggle(std::size_t x);
     void mark_all_but(std::size_t x);
@@ -51,11 +59,17 @@ public:
     DeduperMainWindow();
 
     void setup_menu();
-    void show_images(const std::vector<std::filesystem::path> &fns);
+    void show_images(const std::vector<fs::path> &fns);
     void update_distances(const std::map<std::pair<std::size_t, std::size_t>, double> &d);
     void update_viewstatus(std::size_t cur, std::size_t size);
     void save_list();
     void load_list();
+
+    void scan_dirs(std::vector<std::pair<fs::path, bool>> paths);
+public Q_SLOTS:
+    void create_new();
+    void update_actions();
+    void show_group(size_t gid);
 Q_SIGNALS:
     void next();
     void prev();
diff --git a/qdeduper/pathchooser.cpp b/qdeduper/pathchooser.cpp
new file mode 100644
index 0000000..c08199f
--- /dev/null
+++ b/qdeduper/pathchooser.cpp
@@ -0,0 +1,79 @@
+#include "pathchooser.hpp"
+
+#include <QDialogButtonBox>
+#include <QLabel>
+#include <QDebug>
+#include <QFileDialog>
+#include <QTableView>
+#include <QPushButton>
+#include <QStandardItemModel>
+#include <QVBoxLayout>
+#include <qdialogbuttonbox.h>
+#include <qfiledialog.h>
+#include <qnamespace.h>
+#include <qstandarditemmodel.h>
+
+PathChooser::PathChooser(QWidget *parent) : QDialog(parent)
+{
+    QVBoxLayout *l = new QVBoxLayout();
+    this->setLayout(l);
+    bb = new QDialogButtonBox(QDialogButtonBox::StandardButton::Ok | QDialogButtonBox::StandardButton::Cancel, this);
+    bb->button(QDialogButtonBox::StandardButton::Ok)->setText("Scan");
+
+    QPushButton *pbbrowse = new QPushButton();
+    pbbrowse->setText("Browse...");
+    pbbrowse->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DirOpenIcon));
+    bb->addButton(pbbrowse, QDialogButtonBox::ButtonRole::ActionRole);
+    QObject::connect(pbbrowse, &QPushButton::pressed, this, &PathChooser::add_new);
+
+    QPushButton *pbdelete = new QPushButton();
+    pbdelete->setText("Delete");
+    pbdelete->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DialogDiscardButton));
+    bb->addButton(pbdelete, QDialogButtonBox::ButtonRole::ActionRole);
+    QObject::connect(pbdelete, &QPushButton::pressed, this, &PathChooser::delete_selected);
+    QObject::connect(bb, &QDialogButtonBox::accepted, this, &PathChooser::accept);
+    QObject::connect(bb, &QDialogButtonBox::rejected, this, &PathChooser::reject);
+    im = new QStandardItemModel(this);
+    tv = new QTableView();
+    tv->setModel(im);
+    tv->setSortingEnabled(false);
+    tv->setSelectionMode(QAbstractItemView::SelectionMode::SingleSelection);
+    im->setHorizontalHeaderLabels({"Path", "Recursive?"});
+
+    QLabel *lb = new QLabel("Choose directories to scan");
+    l->addWidget(lb);
+    l->addWidget(tv);
+    l->addWidget(bb);
+}
+
+std::vector<std::pair<fs::path, bool>> PathChooser::get_dirs()
+{
+    std::vector<std::pair<fs::path, bool>> ret;
+    for (int i = 0; i < im->rowCount(); ++i)
+    {
+#if PATH_VALSIZE == 2
+        fs::path p(im->item(i, 0)->text().toStdWString());
+#else
+        fs::path p(im->item(i, 0)->text().toStdString());
+#endif
+        ret.emplace_back(p, (im->item(i, 1)->checkState() == Qt::CheckState::Checked));
+    }
+    return ret;
+}
+
+void PathChooser::add_new()
+{
+    QString s = QFileDialog::getExistingDirectory(this, "Open");
+    if (s.isNull() || s.isEmpty()) return;
+    QStandardItem *it = new QStandardItem(s);
+    QStandardItem *ck = new QStandardItem();
+    it->setEditable(false);
+    ck->setCheckable(true);
+    im->appendRow({it, ck});
+    tv->resizeColumnsToContents();
+}
+
+void PathChooser::delete_selected()
+{
+    im->removeRow(tv->currentIndex().row());
+}
diff --git a/qdeduper/pathchooser.hpp b/qdeduper/pathchooser.hpp
new file mode 100644
index 0000000..07f9f51
--- /dev/null
+++ b/qdeduper/pathchooser.hpp
@@ -0,0 +1,31 @@
+#ifndef PATHCHOOSER_HPP
+#define PATHCHOOSER_HPP
+
+#include <filesystem>
+#include <utility>
+#include <vector>
+
+#include <QDialog>
+
+namespace fs = std::filesystem;
+
+class QDialogButtonBox;
+class QTableView;
+class QStandardItemModel;
+
+class PathChooser : public QDialog
+{
+    Q_OBJECT
+private:
+    QTableView *tv;
+    QStandardItemModel *im;
+    QDialogButtonBox *bb;
+public:
+    PathChooser(QWidget *parent = nullptr);
+    std::vector<std::pair<fs::path, bool>> get_dirs();
+public Q_SLOTS:
+    void add_new();
+    void delete_selected();
+};
+
+#endif
diff --git a/qdeduper/sigdb_qt.cpp b/qdeduper/sigdb_qt.cpp
new file mode 100644
index 0000000..ab6a9f9
--- /dev/null
+++ b/qdeduper/sigdb_qt.cpp
@@ -0,0 +1,122 @@
+#include "sigdb_qt.hpp"
+#include "signature_db.hpp"
+
+#include <algorithm>
+
+signature_config cfg_full =
+{
+    9,      //slices
+    3,      //blur_window
+    2,      //min_window
+    true,   //crop
+    true,   //comp
+    0.5,    //pr
+    1./128, //noise_threshold
+    0.05,   //contrast_threshold
+    0.25    //max_cropping
+};
+
+signature_config cfg_subslice =
+{
+    4,      //slices
+    16,     //blur_window
+    2,      //min_window
+    false,  //crop
+    true,   //comp
+    0.5,    //pr
+    1./64,  //noise_threshold
+    0.05,   //contrast_threshold
+    0.25    //max_cropping
+};
+
+SignatureDB::SignatureDB() : QObject(nullptr)
+{
+    sdb = new signature_db();
+}
+
+SignatureDB::SignatureDB(const fs::path &dbpath) : QObject(nullptr)
+{
+    sdb = new signature_db(dbpath);
+}
+
+SignatureDB::~SignatureDB()
+{
+    delete sdb;
+}
+
+void SignatureDB::scan_files(const std::vector<fs::path> &files, int njobs)
+{
+    populate_cfg_t pcfg = {
+        3,
+        3,
+        cfg_full,
+        cfg_subslice,
+        0.3,
+        [this](size_t c,int){Q_EMIT image_scanned(c);},
+        njobs
+    };
+    sdb->populate(files, pcfg);
+
+    Q_EMIT image_scanned(~size_t(0));
+    
+    auto ids = sdb->get_image_ids();
+    sdb->batch_get_signature_begin();
+    for (auto &id : ids)
+    {
+        fs::path p;
+        std::tie(p, std::ignore) = sdb->get_signature(id);
+        fmap[id] = p;
+        frmap[p] = id;
+    }
+    sdb->batch_get_signature_end();
+
+    auto dupes = sdb->dupe_pairs();
+    for (auto &dupe : dupes)
+        distmap[std::make_pair(dupe.id1, dupe.id2)] = dupe.distance;
+
+    sdb->group_similar();
+    auto gps = sdb->groups_get();
+    gps.erase(std::remove_if(gps.begin(), gps.end(), [](std::vector<size_t> v){ return v.size() < 2; }), gps.end());
+    this->groups = std::move(gps);
+}
+
+size_t SignatureDB::num_groups()
+{
+    return groups.size();
+}
+
+std::vector<size_t> SignatureDB::get_group(size_t gid)
+{
+    if (gid >= groups.size()) return {};
+    return groups[gid];
+}
+
+std::map<std::pair<size_t, size_t>, double> SignatureDB::group_distances(size_t gid)
+{
+    std::map<std::pair<size_t, size_t>, double> ret;
+    auto g = get_group(gid);
+    for (size_t i = 0; i < g.size(); ++i)
+        for (size_t j = i + 1; j < g.size(); ++j)
+        {
+            size_t x = g[i], y = g[j];
+            if (distmap.find(std::make_pair(x, y)) != distmap.end())
+                ret[std::make_pair(i, j)] = distmap[std::make_pair(x, y)];
+            else if (distmap.find(std::make_pair(y, x)) != distmap.end())
+                ret[std::make_pair(j, i)] = distmap[std::make_pair(x, y)];
+        }
+    return ret;
+}
+
+fs::path SignatureDB::get_image_path(size_t id)
+{
+    if (fmap.find(id) == fmap.end())
+        return fs::path();
+    else return fmap[id];
+}
+
+size_t SignatureDB::get_path_id(const fs::path& p)
+{
+    if (frmap.find(p) == frmap.end())
+        return ~size_t(0);
+    else return frmap[p];
+}
diff --git a/qdeduper/sigdb_qt.hpp b/qdeduper/sigdb_qt.hpp
new file mode 100644
index 0000000..8a5178a
--- /dev/null
+++ b/qdeduper/sigdb_qt.hpp
@@ -0,0 +1,41 @@
+#ifndef SIGDB_QT_HPP
+#define SIGDB_QT_HPP
+
+#include <filesystem>
+#include <map>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <QObject>
+
+#include "signature_db.hpp"
+
+namespace fs = std::filesystem;
+
+class SignatureDB : public QObject
+{
+    Q_OBJECT
+private:
+    signature_db *sdb;
+    std::unordered_map<size_t, fs::path> fmap;
+    std::unordered_map<fs::path, size_t> frmap;
+    std::map<std::pair<size_t, size_t>, double> distmap;
+    std::vector<std::vector<size_t>> groups;
+public:
+    SignatureDB();
+    SignatureDB(const fs::path& dbpath);
+    ~SignatureDB();
+
+    void scan_files(const std::vector<fs::path> &files, int njobs);
+    size_t num_groups();
+    std::vector<size_t> get_group(size_t gid);
+    std::map<std::pair<size_t, size_t>, double> group_distances(size_t gid);
+
+    fs::path get_image_path(size_t id);
+    size_t get_path_id(const fs::path& p);
+Q_SIGNALS:
+    void image_scanned(size_t n);
+};
+
+#endif
diff --git a/xsig/include/signature_db.hpp b/xsig/include/signature_db.hpp
index b37cf0a..107aa90 100644
--- a/xsig/include/signature_db.hpp
+++ b/xsig/include/signature_db.hpp
@@ -57,6 +57,8 @@ public:
     std::pair<fs::path, signature> get_signature(size_t id);
     void batch_get_signature_end();
 
+    std::vector<size_t> get_image_ids();
+
     //place batch_put_subslice_begin() and batch_put_subslice_end() around a group of
     //put_subslice() calls to improve performance
     void batch_put_subslice_begin();
diff --git a/xsig/src/signature_db.cpp b/xsig/src/signature_db.cpp
index 393b756..ba1a372 100644
--- a/xsig/src/signature_db.cpp
+++ b/xsig/src/signature_db.cpp
@@ -214,6 +214,22 @@ void signature_db::batch_get_signature_end()
     p->batch_end(batch_status::getsig);
 }
 
+std::vector<size_t> signature_db::get_image_ids()
+{
+    sqlite3_stmt *st = nullptr;
+    sqlite3_prepare_v2(p->db, "select id from images;", -1, &st, 0);
+    std::vector<size_t> ret;
+    while (1)
+    {
+        int r = sqlite3_step(st);
+        if (r != SQLITE_ROW) break;
+        size_t id = (size_t)sqlite3_column_int(st, 0);
+        ret.push_back(id);
+    }
+    sqlite3_finalize(st);
+    return ret;
+}
+
 void signature_db::batch_put_subslice_begin()
 {
     if (!p->db) [[ unlikely ]] return;