aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGravatar Chris Xiong <chirs241097@gmail.com> 2022-09-19 02:39:03 -0400
committerGravatar Chris Xiong <chirs241097@gmail.com> 2022-09-19 02:39:03 -0400
commit41e9051f2d809c42c3dfecc2eb11ad544cbd27b7 (patch)
treee370e08b0e0a45c6eef38704aa2f2b2b0e6d8033
parent4b8d314f575d9e893d8dda7431194f8b470fc888 (diff)
downloaddeduper-41e9051f2d809c42c3dfecc2eb11ad544cbd27b7.tar.xz
You break it, you fix it!
The GUI is now working again, with scanning built-in.
-rw-r--r--.gitignore1
-rw-r--r--qdeduper/CMakeLists.txt6
-rw-r--r--qdeduper/filescanner.cpp75
-rw-r--r--qdeduper/filescanner.hpp30
-rw-r--r--qdeduper/mingui.cpp162
-rw-r--r--qdeduper/mingui.hpp18
-rw-r--r--qdeduper/pathchooser.cpp79
-rw-r--r--qdeduper/pathchooser.hpp31
-rw-r--r--qdeduper/sigdb_qt.cpp122
-rw-r--r--qdeduper/sigdb_qt.hpp41
-rw-r--r--xsig/include/signature_db.hpp2
-rw-r--r--xsig/src/signature_db.cpp16
12 files changed, 553 insertions, 30 deletions
diff --git a/.gitignore b/.gitignore
index cfb2ba4..790e084 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ util_internal/
mingui/build/
CMakeLists.txt.user
+*.swp
diff --git a/qdeduper/CMakeLists.txt b/qdeduper/CMakeLists.txt
index f862c16..270cd45 100644
--- a/qdeduper/CMakeLists.txt
+++ b/qdeduper/CMakeLists.txt
@@ -1,6 +1,6 @@
set(CMAKE_INCLUDE_CURRENT_DIR ON)
-find_package(Qt5 REQUIRED COMPONENTS Widgets)
+find_package(Qt5 REQUIRED COMPONENTS Widgets Concurrent)
set(CMAKE_AUTOMOC ON)
set(CMAKE_AUTORCC ON)
@@ -10,9 +10,13 @@ add_executable(qdeduper
main.cpp
mingui.cpp
imageitem.cpp
+ sigdb_qt.cpp
+ filescanner.cpp
+ pathchooser.cpp
)
target_link_libraries(qdeduper
xsig
Qt5::Widgets
+ Qt5::Concurrent
)
diff --git a/qdeduper/filescanner.cpp b/qdeduper/filescanner.cpp
new file mode 100644
index 0000000..e7e45fc
--- /dev/null
+++ b/qdeduper/filescanner.cpp
@@ -0,0 +1,75 @@
+#include "filescanner.hpp"
+
+#include <cstring>
+#include <algorithm>
+#include <fstream>
+
+using std::size_t;
+
+FileScanner::FileScanner() : QObject(nullptr), maxmnlen(0)
+{
+
+}
+
+void FileScanner::add_magic_number(const std::string &m)
+{
+ mn.push_back(m);
+ if (m.length() > maxmnlen)
+ maxmnlen = m.length();
+}
+
+void FileScanner::add_path(const fs::path &p, bool recurse)
+{
+ paths.emplace_back(p, recurse);
+}
+
+template <class T>
+void dirit_foreach(T iter, std::function<void(const fs::directory_entry& p)> f)
+{
+ std::for_each(fs::begin(iter), fs::end(iter), f);
+}
+
+void FileScanner::scan()
+{
+ size_t fcnt = 0;
+ auto opt = std::filesystem::directory_options::skip_permission_denied;
+ auto count_files = [&fcnt](const fs::directory_entry& e){
+ if (e.is_regular_file()) ++fcnt;
+ };
+ auto scan_file = [&fcnt, this](const fs::directory_entry &e) {
+ if (!e.is_regular_file()) return;
+ std::fstream fst(e.path(), std::ios::binary | std::ios::in);
+ std::string buf(maxmnlen, '\0');
+ fst.read(buf.data(), maxmnlen);
+ buf.resize(fst.gcount());
+ for (auto &magic : mn)
+ if (!memcmp(magic.data(), buf.data(), magic.length()))
+ {
+ ret.push_back(e.path());
+ break;
+ }
+ Q_EMIT file_scanned(e.path(), ++fcnt);
+ };
+ auto for_all_paths = [opt, this](std::function<void(const fs::directory_entry&)> f) {
+ for (auto &pe : paths)
+ {
+ fs::path p;
+ bool recurse;
+ std::tie(p, recurse) = pe;
+ if (recurse)
+ dirit_foreach(fs::recursive_directory_iterator(p, opt), f);
+ else
+ dirit_foreach(fs::directory_iterator(p, opt), f);
+ }
+ };
+ for_all_paths(count_files);
+ Q_EMIT scan_done_prep(fcnt);
+
+ fcnt = 0;
+ for_all_paths(scan_file);
+}
+
+std::vector<fs::path> FileScanner::file_list()
+{
+ return ret;
+}
diff --git a/qdeduper/filescanner.hpp b/qdeduper/filescanner.hpp
new file mode 100644
index 0000000..5d927a4
--- /dev/null
+++ b/qdeduper/filescanner.hpp
@@ -0,0 +1,30 @@
+#ifndef FILESCANNER_HPP
+#define FILESCANNER_HPP
+
+#include <filesystem>
+#include <string>
+#include <utility>
+#include <vector>
+#include <QObject>
+
+namespace fs = std::filesystem;
+
+class FileScanner : public QObject
+{
+ Q_OBJECT
+ std::vector<std::string> mn;
+ std::vector<std::pair<fs::path, bool>> paths;
+ std::vector<fs::path> ret;
+ std::size_t maxmnlen;
+public:
+ FileScanner();
+ void add_magic_number(const std::string &m);
+ void add_path(const fs::path &p, bool recurse = false);
+ void scan();
+ std::vector<fs::path> file_list();
+Q_SIGNALS:
+ void scan_done_prep(std::size_t nfiles);
+ void file_scanned(const fs::path &p, std::size_t n);
+};
+
+#endif
diff --git a/qdeduper/mingui.cpp b/qdeduper/mingui.cpp
index 317076c..e936a36 100644
--- a/qdeduper/mingui.cpp
+++ b/qdeduper/mingui.cpp
@@ -1,11 +1,18 @@
#include "mingui.hpp"
#include "imageitem.hpp"
+#include "filescanner.hpp"
+#include "pathchooser.hpp"
+#include "sigdb_qt.hpp"
#include <cstdio>
+#include <chrono>
#include <cwchar>
+#include <qnamespace.h>
#include <type_traits>
#include <QDebug>
+#include <QtConcurrent>
+#include <QFutureWatcher>
#include <QCloseEvent>
#include <QMouseEvent>
#include <QScrollBar>
@@ -16,6 +23,7 @@
#include <QString>
#include <QScrollArea>
#include <QListView>
+#include <QProgressDialog>
#include <QStandardItemModel>
#include <QLabel>
#include <QHBoxLayout>
@@ -83,6 +91,12 @@ DeduperMainWindow::DeduperMainWindow()
lw->setHorizontalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel);
lw->setVerticalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel);
lw->setMinimumWidth(240);
+ pd = new QProgressDialog(this);
+ pd->setModal(true);
+ pd->setMinimumDuration(0);
+ pd->setAutoReset(false);
+ pd->setAutoClose(false);
+ pd->close();
for (size_t i = 0; i < keys.size(); ++i)
{
@@ -105,29 +119,10 @@ DeduperMainWindow::DeduperMainWindow()
mnone->setShortcut(QKeySequence(Qt::Key::Key_C));
QObject::connect(mnone, &QAction::triggered, [this]{this->mark_none();});
this->addAction(mnone);
- QAction *nxt = new QAction();
- nxt->setShortcut(QKeySequence(Qt::Key::Key_M));
- QObject::connect(nxt, &QAction::triggered, [this]{Q_EMIT this->next();});
- this->addAction(nxt);
- QAction *prv = new QAction();
- prv->setShortcut(QKeySequence(Qt::Key::Key_Z));
- QObject::connect(prv, &QAction::triggered, [this]{Q_EMIT this->prev();});
- this->addAction(prv);
QAction *load = new QAction();
load->setShortcut(QKeySequence(Qt::Key::Key_N));
QObject::connect(load, &QAction::triggered, [this]{Q_EMIT this->load_list();});
this->addAction(load);
- QAction *skip = new QAction();
- skip->setShortcut(QKeySequence(Qt::Key::Key_B));
- QObject::connect(skip, &QAction::triggered, [this]{
- bool ok = false;
- int g = QInputDialog::getInt(this, "Skip to group",
- QString("Group # (1-%1)").arg(ngroups),
- curgroup + 1,
- 1, ngroups, 1, &ok);
- if (ok) Q_EMIT switch_group((size_t) g - 1);
- });
- this->addAction(skip);
QAction *save = new QAction();
save->setShortcut(QKeySequence(Qt::Modifier::SHIFT | Qt::Key::Key_Return));
QObject::connect(save, &QAction::triggered, [this]{Q_EMIT this->save_list();});
@@ -173,7 +168,9 @@ void DeduperMainWindow::setup_menu()
QMenu *mark = this->menuBar()->addMenu("Marks");
QMenu *help = this->menuBar()->addMenu("Help");
- file->addAction("Create Database...");
+ QAction *create_db = file->addAction("Create Database...");
+ QObject::connect(create_db, &QAction::triggered, this, &DeduperMainWindow::create_new);
+ menuact["create_db"] = create_db;
file->addAction("Load Database...");
file->addAction("Save Database...");
file->addSeparator();
@@ -182,8 +179,38 @@ void DeduperMainWindow::setup_menu()
file->addAction("Preferences...");
file->addAction("Exit");
- view->addAction("Next Group");
- view->addAction("Previous Group");
+ QAction *nxtgrp = view->addAction("Next Group");
+ menuact["next_group"] = nxtgrp;
+ nxtgrp->setShortcut(QKeySequence(Qt::Key::Key_M));
+ QObject::connect(nxtgrp, &QAction::triggered, [this] {
+ if (this->sdb && curgroup + 1 < this->sdb->num_groups())
+ this->show_group(++curgroup);
+ });
+ this->addAction(nxtgrp);
+
+ QAction *prvgrp = view->addAction("Previous Group");
+ menuact["prev_group"] = prvgrp;
+ prvgrp->setShortcut(QKeySequence(Qt::Key::Key_Z));
+ QObject::connect(prvgrp, &QAction::triggered, [this] {
+ if (this->sdb && curgroup > 1)
+ this->show_group(--curgroup);
+ });
+ this->addAction(prvgrp);
+
+ QAction *skip = view->addAction("Skip to Group...");
+ menuact["skip_group"] = skip;
+ skip->setShortcut(QKeySequence(Qt::Key::Key_B));
+ QObject::connect(skip, &QAction::triggered, [this] {
+ if (!this->sdb) return;
+ bool ok = false;
+ int g = QInputDialog::getInt(this, "Skip to group",
+ QString("Group # (1-%1)").arg(sdb->num_groups()),
+ curgroup + 1,
+ 1, sdb->num_groups(), 1, &ok);
+ if (ok) this->show_group((size_t) g - 1);
+ });
+ this->addAction(skip);
+
view->addSeparator();
QMenu *sort = view->addMenu("Sort by");
sort->addAction("File size");
@@ -198,16 +225,21 @@ void DeduperMainWindow::setup_menu()
help->addAction("View Documentation");
help->addAction("About");
}
+void DeduperMainWindow::update_actions()
+{
+ if (!sdb)
+ {
+ menuact["next_group"]->setEnabled(false);
+ menuact["prev_group"]->setEnabled(false);
+ return;
+ }
+}
void DeduperMainWindow::show_images(const std::vector<fs::path> &fns)
{
current_set = fns;
marks.clear();
im->clear();
- int max_height = (this->screen()->size().height() / fns.size() * 0.8 - 24) * this->screen()->devicePixelRatio();
- int max_width = this->screen()->size().width() * 0.8 * this->screen()->devicePixelRatio();
- if (max_height < 64) max_height = 64;
- if (max_width < 64) max_width = 64;
fs::path::string_type common_pfx = common_prefix(fns);
size_t idx = 0;
if (fns.size() > keys.size() && !nohotkeywarn)
@@ -245,7 +277,6 @@ void DeduperMainWindow::update_distances(const std::map<std::pair<size_t, size_t
void DeduperMainWindow::update_viewstatus(std::size_t cur, std::size_t size)
{
permamsg->setText(QString("Viewing group %1 of %2").arg(cur + 1).arg(size));
- ngroups = size;
curgroup = cur;
}
@@ -291,6 +322,83 @@ void DeduperMainWindow::load_list()
mark_view_update();
}
+void DeduperMainWindow::create_new()
+{
+ PathChooser *pc = new PathChooser(this);
+ pc->setModal(true);
+ if (pc->exec() == QDialog::DialogCode::Accepted)
+ this->scan_dirs(pc->get_dirs());
+ pc->deleteLater();
+}
+
+void DeduperMainWindow::scan_dirs(std::vector<std::pair<fs::path, bool>> paths)
+{
+ this->pd->open();
+ this->pd->setLabelText("Preparing for database creation...");
+ this->pd->setMinimum(0);
+ this->pd->setMaximum(0);
+ auto f = QtConcurrent::run([this, paths] {
+ FileScanner *fs = new FileScanner();
+ std::for_each(paths.begin(), paths.end(), [fs](auto p){fs->add_path(p.first, p.second);});
+ fs->add_magic_number("\x89PNG\r\n");
+ fs->add_magic_number("\xff\xd8\xff");
+ QObject::connect(fs, &FileScanner::scan_done_prep, [this](auto n) {
+ this->pd->setMaximum(n - 1);
+ });
+ QObject::connect(fs, &FileScanner::file_scanned, [this](const fs::path &p, size_t c) {
+ static auto lt = std::chrono::steady_clock::now();
+ using namespace std::literals;
+ if (std::chrono::steady_clock::now() - lt > 100ms)
+ {
+ lt = std::chrono::steady_clock::now();
+ this->pd->setLabelText(QString("Looking for files to scan: %1").arg(fsstr_to_qstring(p)));
+ this->pd->setValue(c);
+ }
+ });
+ fs->scan();
+ this->pd->setMaximum(fs->file_list().size() - 1);
+ this->pd->setLabelText("Scanning...");
+ this->sdb = new SignatureDB();
+ QObject::connect(this->sdb, &SignatureDB::image_scanned, this, [this](size_t n) {
+ static auto lt = std::chrono::steady_clock::now();
+ using namespace std::literals;
+ if (std::chrono::steady_clock::now() - lt > 100ms)
+ {
+ lt = std::chrono::steady_clock::now();
+ this->pd->setLabelText(QString("Scanning %1 / %2").arg(n + 1).arg(this->pd->maximum() + 1));
+ this->pd->setValue(n);
+ }
+ if (!~n)
+ {
+ this->pd->setMaximum(0);
+ this->pd->setLabelText("Finalizing...");
+ }
+ }, Qt::ConnectionType::QueuedConnection);
+ this->sdb->scan_files(fs->file_list(), 8);
+ delete fs;
+ });
+ QFutureWatcher<void> *fw = new QFutureWatcher<void>(this);
+ fw->setFuture(f);
+ QObject::connect(fw, &QFutureWatcher<void>::finished, this, [this] {
+ this->pd->reset();
+ this->pd->close();
+ this->curgroup = 0;
+ this->show_group(this->curgroup);
+ }, Qt::ConnectionType::QueuedConnection);
+}
+
+void DeduperMainWindow::show_group(size_t gid)
+{
+ if (!sdb || gid >= sdb->num_groups())
+ return;
+ auto g = sdb->get_group(gid);
+ current_set.clear();
+ std::for_each(g.begin(), g.end(), [this](auto id){current_set.push_back(sdb->get_image_path(id));});
+ this->show_images(current_set);
+ this->update_distances(sdb->group_distances(gid));
+ this->update_viewstatus(gid, sdb->num_groups());
+}
+
void DeduperMainWindow::mark_toggle(size_t x)
{
if (x < marks.size())
diff --git a/qdeduper/mingui.hpp b/qdeduper/mingui.hpp
index b8d13ef..3515047 100644
--- a/qdeduper/mingui.hpp
+++ b/qdeduper/mingui.hpp
@@ -2,6 +2,7 @@
#define MINGUI_HPP
#include <filesystem>
+#include <map>
#include <vector>
#include <string>
#include <unordered_set>
@@ -9,12 +10,15 @@
#include <QMainWindow>
#include <QList>
+#include "sigdb_qt.hpp"
+
class QHBoxLayout;
class QLabel;
class QStatusBar;
class QScrollArea;
class QTextEdit;
class QListView;
+class QProgressDialog;
class QSplitter;
class QStandardItemModel;
class ImageItemDelegate;
@@ -30,10 +34,14 @@ private:
QLabel *permamsg;
QStatusBar *sb;
QListView *lw;
+ std::map<std::string, QAction*> menuact;
QList<QAction*> selhk;
QStandardItemModel *im = nullptr;
ImageItemDelegate *id = nullptr;
- std::size_t ngroups, curgroup;
+ QProgressDialog *pd = nullptr;
+ SignatureDB *sdb = nullptr;
+
+ std::size_t curgroup;
bool nohotkeywarn;
void mark_toggle(std::size_t x);
void mark_all_but(std::size_t x);
@@ -51,11 +59,17 @@ public:
DeduperMainWindow();
void setup_menu();
- void show_images(const std::vector<std::filesystem::path> &fns);
+ void show_images(const std::vector<fs::path> &fns);
void update_distances(const std::map<std::pair<std::size_t, std::size_t>, double> &d);
void update_viewstatus(std::size_t cur, std::size_t size);
void save_list();
void load_list();
+
+ void scan_dirs(std::vector<std::pair<fs::path, bool>> paths);
+public Q_SLOTS:
+ void create_new();
+ void update_actions();
+ void show_group(size_t gid);
Q_SIGNALS:
void next();
void prev();
diff --git a/qdeduper/pathchooser.cpp b/qdeduper/pathchooser.cpp
new file mode 100644
index 0000000..c08199f
--- /dev/null
+++ b/qdeduper/pathchooser.cpp
@@ -0,0 +1,79 @@
+#include "pathchooser.hpp"
+
+#include <QDialogButtonBox>
+#include <QLabel>
+#include <QDebug>
+#include <QFileDialog>
+#include <QTableView>
+#include <QPushButton>
+#include <QStandardItemModel>
+#include <QVBoxLayout>
+#include <qdialogbuttonbox.h>
+#include <qfiledialog.h>
+#include <qnamespace.h>
+#include <qstandarditemmodel.h>
+
+PathChooser::PathChooser(QWidget *parent) : QDialog(parent)
+{
+ QVBoxLayout *l = new QVBoxLayout();
+ this->setLayout(l);
+ bb = new QDialogButtonBox(QDialogButtonBox::StandardButton::Ok | QDialogButtonBox::StandardButton::Cancel, this);
+ bb->button(QDialogButtonBox::StandardButton::Ok)->setText("Scan");
+
+ QPushButton *pbbrowse = new QPushButton();
+ pbbrowse->setText("Browse...");
+ pbbrowse->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DirOpenIcon));
+ bb->addButton(pbbrowse, QDialogButtonBox::ButtonRole::ActionRole);
+ QObject::connect(pbbrowse, &QPushButton::pressed, this, &PathChooser::add_new);
+
+ QPushButton *pbdelete = new QPushButton();
+ pbdelete->setText("Delete");
+ pbdelete->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DialogDiscardButton));
+ bb->addButton(pbdelete, QDialogButtonBox::ButtonRole::ActionRole);
+ QObject::connect(pbdelete, &QPushButton::pressed, this, &PathChooser::delete_selected);
+ QObject::connect(bb, &QDialogButtonBox::accepted, this, &PathChooser::accept);
+ QObject::connect(bb, &QDialogButtonBox::rejected, this, &PathChooser::reject);
+ im = new QStandardItemModel(this);
+ tv = new QTableView();
+ tv->setModel(im);
+ tv->setSortingEnabled(false);
+ tv->setSelectionMode(QAbstractItemView::SelectionMode::SingleSelection);
+ im->setHorizontalHeaderLabels({"Path", "Recursive?"});
+
+ QLabel *lb = new QLabel("Choose directories to scan");
+ l->addWidget(lb);
+ l->addWidget(tv);
+ l->addWidget(bb);
+}
+
+std::vector<std::pair<fs::path, bool>> PathChooser::get_dirs()
+{
+ std::vector<std::pair<fs::path, bool>> ret;
+ for (int i = 0; i < im->rowCount(); ++i)
+ {
+#if PATH_VALSIZE == 2
+ fs::path p(im->item(i, 0)->text().toStdWString());
+#else
+ fs::path p(im->item(i, 0)->text().toStdString());
+#endif
+ ret.emplace_back(p, (im->item(i, 1)->checkState() == Qt::CheckState::Checked));
+ }
+ return ret;
+}
+
+void PathChooser::add_new()
+{
+ QString s = QFileDialog::getExistingDirectory(this, "Open");
+ if (s.isNull() || s.isEmpty()) return;
+ QStandardItem *it = new QStandardItem(s);
+ QStandardItem *ck = new QStandardItem();
+ it->setEditable(false);
+ ck->setCheckable(true);
+ im->appendRow({it, ck});
+ tv->resizeColumnsToContents();
+}
+
+void PathChooser::delete_selected()
+{
+ im->removeRow(tv->currentIndex().row());
+}
diff --git a/qdeduper/pathchooser.hpp b/qdeduper/pathchooser.hpp
new file mode 100644
index 0000000..07f9f51
--- /dev/null
+++ b/qdeduper/pathchooser.hpp
@@ -0,0 +1,31 @@
+#ifndef PATHCHOOSER_HPP
+#define PATHCHOOSER_HPP
+
+#include <filesystem>
+#include <utility>
+#include <vector>
+
+#include <QDialog>
+
+namespace fs = std::filesystem;
+
+class QDialogButtonBox;
+class QTableView;
+class QStandardItemModel;
+
+class PathChooser : public QDialog
+{
+ Q_OBJECT
+private:
+ QTableView *tv;
+ QStandardItemModel *im;
+ QDialogButtonBox *bb;
+public:
+ PathChooser(QWidget *parent = nullptr);
+ std::vector<std::pair<fs::path, bool>> get_dirs();
+public Q_SLOTS:
+ void add_new();
+ void delete_selected();
+};
+
+#endif
diff --git a/qdeduper/sigdb_qt.cpp b/qdeduper/sigdb_qt.cpp
new file mode 100644
index 0000000..ab6a9f9
--- /dev/null
+++ b/qdeduper/sigdb_qt.cpp
@@ -0,0 +1,122 @@
+#include "sigdb_qt.hpp"
+#include "signature_db.hpp"
+
+#include <algorithm>
+
+signature_config cfg_full =
+{
+ 9, //slices
+ 3, //blur_window
+ 2, //min_window
+ true, //crop
+ true, //comp
+ 0.5, //pr
+ 1./128, //noise_threshold
+ 0.05, //contrast_threshold
+ 0.25 //max_cropping
+};
+
+signature_config cfg_subslice =
+{
+ 4, //slices
+ 16, //blur_window
+ 2, //min_window
+ false, //crop
+ true, //comp
+ 0.5, //pr
+ 1./64, //noise_threshold
+ 0.05, //contrast_threshold
+ 0.25 //max_cropping
+};
+
+SignatureDB::SignatureDB() : QObject(nullptr)
+{
+ sdb = new signature_db();
+}
+
+SignatureDB::SignatureDB(const fs::path &dbpath) : QObject(nullptr)
+{
+ sdb = new signature_db(dbpath);
+}
+
+SignatureDB::~SignatureDB()
+{
+ delete sdb;
+}
+
+void SignatureDB::scan_files(const std::vector<fs::path> &files, int njobs)
+{
+ populate_cfg_t pcfg = {
+ 3,
+ 3,
+ cfg_full,
+ cfg_subslice,
+ 0.3,
+ [this](size_t c,int){Q_EMIT image_scanned(c);},
+ njobs
+ };
+ sdb->populate(files, pcfg);
+
+ Q_EMIT image_scanned(~size_t(0));
+
+ auto ids = sdb->get_image_ids();
+ sdb->batch_get_signature_begin();
+ for (auto &id : ids)
+ {
+ fs::path p;
+ std::tie(p, std::ignore) = sdb->get_signature(id);
+ fmap[id] = p;
+ frmap[p] = id;
+ }
+ sdb->batch_get_signature_end();
+
+ auto dupes = sdb->dupe_pairs();
+ for (auto &dupe : dupes)
+ distmap[std::make_pair(dupe.id1, dupe.id2)] = dupe.distance;
+
+ sdb->group_similar();
+ auto gps = sdb->groups_get();
+ gps.erase(std::remove_if(gps.begin(), gps.end(), [](std::vector<size_t> v){ return v.size() < 2; }), gps.end());
+ this->groups = std::move(gps);
+}
+
+size_t SignatureDB::num_groups()
+{
+ return groups.size();
+}
+
+std::vector<size_t> SignatureDB::get_group(size_t gid)
+{
+ if (gid >= groups.size()) return {};
+ return groups[gid];
+}
+
+std::map<std::pair<size_t, size_t>, double> SignatureDB::group_distances(size_t gid)
+{
+ std::map<std::pair<size_t, size_t>, double> ret;
+ auto g = get_group(gid);
+ for (size_t i = 0; i < g.size(); ++i)
+ for (size_t j = i + 1; j < g.size(); ++j)
+ {
+ size_t x = g[i], y = g[j];
+ if (distmap.find(std::make_pair(x, y)) != distmap.end())
+ ret[std::make_pair(i, j)] = distmap[std::make_pair(x, y)];
+ else if (distmap.find(std::make_pair(y, x)) != distmap.end())
+ ret[std::make_pair(j, i)] = distmap[std::make_pair(x, y)];
+ }
+ return ret;
+}
+
+fs::path SignatureDB::get_image_path(size_t id)
+{
+ if (fmap.find(id) == fmap.end())
+ return fs::path();
+ else return fmap[id];
+}
+
+size_t SignatureDB::get_path_id(const fs::path& p)
+{
+ if (frmap.find(p) == frmap.end())
+ return ~size_t(0);
+ else return frmap[p];
+}
diff --git a/qdeduper/sigdb_qt.hpp b/qdeduper/sigdb_qt.hpp
new file mode 100644
index 0000000..8a5178a
--- /dev/null
+++ b/qdeduper/sigdb_qt.hpp
@@ -0,0 +1,41 @@
+#ifndef SIGDB_QT_HPP
+#define SIGDB_QT_HPP
+
+#include <filesystem>
+#include <map>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include <QObject>
+
+#include "signature_db.hpp"
+
+namespace fs = std::filesystem;
+
+class SignatureDB : public QObject
+{
+ Q_OBJECT
+private:
+ signature_db *sdb;
+ std::unordered_map<size_t, fs::path> fmap;
+ std::unordered_map<fs::path, size_t> frmap;
+ std::map<std::pair<size_t, size_t>, double> distmap;
+ std::vector<std::vector<size_t>> groups;
+public:
+ SignatureDB();
+ SignatureDB(const fs::path& dbpath);
+ ~SignatureDB();
+
+ void scan_files(const std::vector<fs::path> &files, int njobs);
+ size_t num_groups();
+ std::vector<size_t> get_group(size_t gid);
+ std::map<std::pair<size_t, size_t>, double> group_distances(size_t gid);
+
+ fs::path get_image_path(size_t id);
+ size_t get_path_id(const fs::path& p);
+Q_SIGNALS:
+ void image_scanned(size_t n);
+};
+
+#endif
diff --git a/xsig/include/signature_db.hpp b/xsig/include/signature_db.hpp
index b37cf0a..107aa90 100644
--- a/xsig/include/signature_db.hpp
+++ b/xsig/include/signature_db.hpp
@@ -57,6 +57,8 @@ public:
std::pair<fs::path, signature> get_signature(size_t id);
void batch_get_signature_end();
+ std::vector<size_t> get_image_ids();
+
//place batch_put_subslice_begin() and batch_put_subslice_end() around a group of
//put_subslice() calls to improve performance
void batch_put_subslice_begin();
diff --git a/xsig/src/signature_db.cpp b/xsig/src/signature_db.cpp
index 393b756..ba1a372 100644
--- a/xsig/src/signature_db.cpp
+++ b/xsig/src/signature_db.cpp
@@ -214,6 +214,22 @@ void signature_db::batch_get_signature_end()
p->batch_end(batch_status::getsig);
}
+std::vector<size_t> signature_db::get_image_ids()
+{
+ sqlite3_stmt *st = nullptr;
+ sqlite3_prepare_v2(p->db, "select id from images;", -1, &st, 0);
+ std::vector<size_t> ret;
+ while (1)
+ {
+ int r = sqlite3_step(st);
+ if (r != SQLITE_ROW) break;
+ size_t id = (size_t)sqlite3_column_int(st, 0);
+ ret.push_back(id);
+ }
+ sqlite3_finalize(st);
+ return ret;
+}
+
void signature_db::batch_put_subslice_begin()
{
if (!p->db) [[ unlikely ]] return;