diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | qdeduper/CMakeLists.txt | 6 | ||||
| -rw-r--r-- | qdeduper/filescanner.cpp | 75 | ||||
| -rw-r--r-- | qdeduper/filescanner.hpp | 30 | ||||
| -rw-r--r-- | qdeduper/mingui.cpp | 162 | ||||
| -rw-r--r-- | qdeduper/mingui.hpp | 18 | ||||
| -rw-r--r-- | qdeduper/pathchooser.cpp | 79 | ||||
| -rw-r--r-- | qdeduper/pathchooser.hpp | 31 | ||||
| -rw-r--r-- | qdeduper/sigdb_qt.cpp | 122 | ||||
| -rw-r--r-- | qdeduper/sigdb_qt.hpp | 41 | ||||
| -rw-r--r-- | xsig/include/signature_db.hpp | 2 | ||||
| -rw-r--r-- | xsig/src/signature_db.cpp | 16 |
12 files changed, 553 insertions, 30 deletions
@@ -5,3 +5,4 @@ util_internal/ mingui/build/ CMakeLists.txt.user +*.swp diff --git a/qdeduper/CMakeLists.txt b/qdeduper/CMakeLists.txt index f862c16..270cd45 100644 --- a/qdeduper/CMakeLists.txt +++ b/qdeduper/CMakeLists.txt @@ -1,6 +1,6 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON) -find_package(Qt5 REQUIRED COMPONENTS Widgets) +find_package(Qt5 REQUIRED COMPONENTS Widgets Concurrent) set(CMAKE_AUTOMOC ON) set(CMAKE_AUTORCC ON) @@ -10,9 +10,13 @@ add_executable(qdeduper main.cpp mingui.cpp imageitem.cpp + sigdb_qt.cpp + filescanner.cpp + pathchooser.cpp ) target_link_libraries(qdeduper xsig Qt5::Widgets + Qt5::Concurrent ) diff --git a/qdeduper/filescanner.cpp b/qdeduper/filescanner.cpp new file mode 100644 index 0000000..e7e45fc --- /dev/null +++ b/qdeduper/filescanner.cpp @@ -0,0 +1,75 @@ +#include "filescanner.hpp" + +#include <cstring> +#include <algorithm> +#include <fstream> + +using std::size_t; + +FileScanner::FileScanner() : QObject(nullptr), maxmnlen(0) +{ + +} + +void FileScanner::add_magic_number(const std::string &m) +{ + mn.push_back(m); + if (m.length() > maxmnlen) + maxmnlen = m.length(); +} + +void FileScanner::add_path(const fs::path &p, bool recurse) +{ + paths.emplace_back(p, recurse); +} + +template <class T> +void dirit_foreach(T iter, std::function<void(const fs::directory_entry& p)> f) +{ + std::for_each(fs::begin(iter), fs::end(iter), f); +} + +void FileScanner::scan() +{ + size_t fcnt = 0; + auto opt = std::filesystem::directory_options::skip_permission_denied; + auto count_files = [&fcnt](const fs::directory_entry& e){ + if (e.is_regular_file()) ++fcnt; + }; + auto scan_file = [&fcnt, this](const fs::directory_entry &e) { + if (!e.is_regular_file()) return; + std::fstream fst(e.path(), std::ios::binary | std::ios::in); + std::string buf(maxmnlen, '\0'); + fst.read(buf.data(), maxmnlen); + buf.resize(fst.gcount()); + for (auto &magic : mn) + if (!memcmp(magic.data(), buf.data(), magic.length())) + { + ret.push_back(e.path()); + break; + } + Q_EMIT file_scanned(e.path(), ++fcnt); + }; + auto for_all_paths = [opt, this](std::function<void(const fs::directory_entry&)> f) { + for (auto &pe : paths) + { + fs::path p; + bool recurse; + std::tie(p, recurse) = pe; + if (recurse) + dirit_foreach(fs::recursive_directory_iterator(p, opt), f); + else + dirit_foreach(fs::directory_iterator(p, opt), f); + } + }; + for_all_paths(count_files); + Q_EMIT scan_done_prep(fcnt); + + fcnt = 0; + for_all_paths(scan_file); +} + +std::vector<fs::path> FileScanner::file_list() +{ + return ret; +} diff --git a/qdeduper/filescanner.hpp b/qdeduper/filescanner.hpp new file mode 100644 index 0000000..5d927a4 --- /dev/null +++ b/qdeduper/filescanner.hpp @@ -0,0 +1,30 @@ +#ifndef FILESCANNER_HPP +#define FILESCANNER_HPP + +#include <filesystem> +#include <string> +#include <utility> +#include <vector> +#include <QObject> + +namespace fs = std::filesystem; + +class FileScanner : public QObject +{ + Q_OBJECT + std::vector<std::string> mn; + std::vector<std::pair<fs::path, bool>> paths; + std::vector<fs::path> ret; + std::size_t maxmnlen; +public: + FileScanner(); + void add_magic_number(const std::string &m); + void add_path(const fs::path &p, bool recurse = false); + void scan(); + std::vector<fs::path> file_list(); +Q_SIGNALS: + void scan_done_prep(std::size_t nfiles); + void file_scanned(const fs::path &p, std::size_t n); +}; + +#endif diff --git a/qdeduper/mingui.cpp b/qdeduper/mingui.cpp index 317076c..e936a36 100644 --- a/qdeduper/mingui.cpp +++ b/qdeduper/mingui.cpp @@ -1,11 +1,18 @@ #include "mingui.hpp" #include "imageitem.hpp" +#include "filescanner.hpp" +#include "pathchooser.hpp" +#include "sigdb_qt.hpp" #include <cstdio> +#include <chrono> #include <cwchar> +#include <qnamespace.h> #include <type_traits> #include <QDebug> +#include <QtConcurrent> +#include <QFutureWatcher> #include <QCloseEvent> #include <QMouseEvent> #include <QScrollBar> @@ -16,6 +23,7 @@ #include <QString> #include <QScrollArea> #include <QListView> +#include <QProgressDialog> #include <QStandardItemModel> #include <QLabel> #include <QHBoxLayout> @@ -83,6 +91,12 @@ DeduperMainWindow::DeduperMainWindow() lw->setHorizontalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel); lw->setVerticalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel); lw->setMinimumWidth(240); + pd = new QProgressDialog(this); + pd->setModal(true); + pd->setMinimumDuration(0); + pd->setAutoReset(false); + pd->setAutoClose(false); + pd->close(); for (size_t i = 0; i < keys.size(); ++i) { @@ -105,29 +119,10 @@ DeduperMainWindow::DeduperMainWindow() mnone->setShortcut(QKeySequence(Qt::Key::Key_C)); QObject::connect(mnone, &QAction::triggered, [this]{this->mark_none();}); this->addAction(mnone); - QAction *nxt = new QAction(); - nxt->setShortcut(QKeySequence(Qt::Key::Key_M)); - QObject::connect(nxt, &QAction::triggered, [this]{Q_EMIT this->next();}); - this->addAction(nxt); - QAction *prv = new QAction(); - prv->setShortcut(QKeySequence(Qt::Key::Key_Z)); - QObject::connect(prv, &QAction::triggered, [this]{Q_EMIT this->prev();}); - this->addAction(prv); QAction *load = new QAction(); load->setShortcut(QKeySequence(Qt::Key::Key_N)); QObject::connect(load, &QAction::triggered, [this]{Q_EMIT this->load_list();}); this->addAction(load); - QAction *skip = new QAction(); - skip->setShortcut(QKeySequence(Qt::Key::Key_B)); - QObject::connect(skip, &QAction::triggered, [this]{ - bool ok = false; - int g = QInputDialog::getInt(this, "Skip to group", - QString("Group # (1-%1)").arg(ngroups), - curgroup + 1, - 1, ngroups, 1, &ok); - if (ok) Q_EMIT switch_group((size_t) g - 1); - }); - this->addAction(skip); QAction *save = new QAction(); save->setShortcut(QKeySequence(Qt::Modifier::SHIFT | Qt::Key::Key_Return)); QObject::connect(save, &QAction::triggered, [this]{Q_EMIT this->save_list();}); @@ -173,7 +168,9 @@ void DeduperMainWindow::setup_menu() QMenu *mark = this->menuBar()->addMenu("Marks"); QMenu *help = this->menuBar()->addMenu("Help"); - file->addAction("Create Database..."); + QAction *create_db = file->addAction("Create Database..."); + QObject::connect(create_db, &QAction::triggered, this, &DeduperMainWindow::create_new); + menuact["create_db"] = create_db; file->addAction("Load Database..."); file->addAction("Save Database..."); file->addSeparator(); @@ -182,8 +179,38 @@ void DeduperMainWindow::setup_menu() file->addAction("Preferences..."); file->addAction("Exit"); - view->addAction("Next Group"); - view->addAction("Previous Group"); + QAction *nxtgrp = view->addAction("Next Group"); + menuact["next_group"] = nxtgrp; + nxtgrp->setShortcut(QKeySequence(Qt::Key::Key_M)); + QObject::connect(nxtgrp, &QAction::triggered, [this] { + if (this->sdb && curgroup + 1 < this->sdb->num_groups()) + this->show_group(++curgroup); + }); + this->addAction(nxtgrp); + + QAction *prvgrp = view->addAction("Previous Group"); + menuact["prev_group"] = prvgrp; + prvgrp->setShortcut(QKeySequence(Qt::Key::Key_Z)); + QObject::connect(prvgrp, &QAction::triggered, [this] { + if (this->sdb && curgroup > 1) + this->show_group(--curgroup); + }); + this->addAction(prvgrp); + + QAction *skip = view->addAction("Skip to Group..."); + menuact["skip_group"] = skip; + skip->setShortcut(QKeySequence(Qt::Key::Key_B)); + QObject::connect(skip, &QAction::triggered, [this] { + if (!this->sdb) return; + bool ok = false; + int g = QInputDialog::getInt(this, "Skip to group", + QString("Group # (1-%1)").arg(sdb->num_groups()), + curgroup + 1, + 1, sdb->num_groups(), 1, &ok); + if (ok) this->show_group((size_t) g - 1); + }); + this->addAction(skip); + view->addSeparator(); QMenu *sort = view->addMenu("Sort by"); sort->addAction("File size"); @@ -198,16 +225,21 @@ void DeduperMainWindow::setup_menu() help->addAction("View Documentation"); help->addAction("About"); } +void DeduperMainWindow::update_actions() +{ + if (!sdb) + { + menuact["next_group"]->setEnabled(false); + menuact["prev_group"]->setEnabled(false); + return; + } +} void DeduperMainWindow::show_images(const std::vector<fs::path> &fns) { current_set = fns; marks.clear(); im->clear(); - int max_height = (this->screen()->size().height() / fns.size() * 0.8 - 24) * this->screen()->devicePixelRatio(); - int max_width = this->screen()->size().width() * 0.8 * this->screen()->devicePixelRatio(); - if (max_height < 64) max_height = 64; - if (max_width < 64) max_width = 64; fs::path::string_type common_pfx = common_prefix(fns); size_t idx = 0; if (fns.size() > keys.size() && !nohotkeywarn) @@ -245,7 +277,6 @@ void DeduperMainWindow::update_distances(const std::map<std::pair<size_t, size_t void DeduperMainWindow::update_viewstatus(std::size_t cur, std::size_t size) { permamsg->setText(QString("Viewing group %1 of %2").arg(cur + 1).arg(size)); - ngroups = size; curgroup = cur; } @@ -291,6 +322,83 @@ void DeduperMainWindow::load_list() mark_view_update(); } +void DeduperMainWindow::create_new() +{ + PathChooser *pc = new PathChooser(this); + pc->setModal(true); + if (pc->exec() == QDialog::DialogCode::Accepted) + this->scan_dirs(pc->get_dirs()); + pc->deleteLater(); +} + +void DeduperMainWindow::scan_dirs(std::vector<std::pair<fs::path, bool>> paths) +{ + this->pd->open(); + this->pd->setLabelText("Preparing for database creation..."); + this->pd->setMinimum(0); + this->pd->setMaximum(0); + auto f = QtConcurrent::run([this, paths] { + FileScanner *fs = new FileScanner(); + std::for_each(paths.begin(), paths.end(), [fs](auto p){fs->add_path(p.first, p.second);}); + fs->add_magic_number("\x89PNG\r\n"); + fs->add_magic_number("\xff\xd8\xff"); + QObject::connect(fs, &FileScanner::scan_done_prep, [this](auto n) { + this->pd->setMaximum(n - 1); + }); + QObject::connect(fs, &FileScanner::file_scanned, [this](const fs::path &p, size_t c) { + static auto lt = std::chrono::steady_clock::now(); + using namespace std::literals; + if (std::chrono::steady_clock::now() - lt > 100ms) + { + lt = std::chrono::steady_clock::now(); + this->pd->setLabelText(QString("Looking for files to scan: %1").arg(fsstr_to_qstring(p))); + this->pd->setValue(c); + } + }); + fs->scan(); + this->pd->setMaximum(fs->file_list().size() - 1); + this->pd->setLabelText("Scanning..."); + this->sdb = new SignatureDB(); + QObject::connect(this->sdb, &SignatureDB::image_scanned, this, [this](size_t n) { + static auto lt = std::chrono::steady_clock::now(); + using namespace std::literals; + if (std::chrono::steady_clock::now() - lt > 100ms) + { + lt = std::chrono::steady_clock::now(); + this->pd->setLabelText(QString("Scanning %1 / %2").arg(n + 1).arg(this->pd->maximum() + 1)); + this->pd->setValue(n); + } + if (!~n) + { + this->pd->setMaximum(0); + this->pd->setLabelText("Finalizing..."); + } + }, Qt::ConnectionType::QueuedConnection); + this->sdb->scan_files(fs->file_list(), 8); + delete fs; + }); + QFutureWatcher<void> *fw = new QFutureWatcher<void>(this); + fw->setFuture(f); + QObject::connect(fw, &QFutureWatcher<void>::finished, this, [this] { + this->pd->reset(); + this->pd->close(); + this->curgroup = 0; + this->show_group(this->curgroup); + }, Qt::ConnectionType::QueuedConnection); +} + +void DeduperMainWindow::show_group(size_t gid) +{ + if (!sdb || gid >= sdb->num_groups()) + return; + auto g = sdb->get_group(gid); + current_set.clear(); + std::for_each(g.begin(), g.end(), [this](auto id){current_set.push_back(sdb->get_image_path(id));}); + this->show_images(current_set); + this->update_distances(sdb->group_distances(gid)); + this->update_viewstatus(gid, sdb->num_groups()); +} + void DeduperMainWindow::mark_toggle(size_t x) { if (x < marks.size()) diff --git a/qdeduper/mingui.hpp b/qdeduper/mingui.hpp index b8d13ef..3515047 100644 --- a/qdeduper/mingui.hpp +++ b/qdeduper/mingui.hpp @@ -2,6 +2,7 @@ #define MINGUI_HPP #include <filesystem> +#include <map> #include <vector> #include <string> #include <unordered_set> @@ -9,12 +10,15 @@ #include <QMainWindow> #include <QList> +#include "sigdb_qt.hpp" + class QHBoxLayout; class QLabel; class QStatusBar; class QScrollArea; class QTextEdit; class QListView; +class QProgressDialog; class QSplitter; class QStandardItemModel; class ImageItemDelegate; @@ -30,10 +34,14 @@ private: QLabel *permamsg; QStatusBar *sb; QListView *lw; + std::map<std::string, QAction*> menuact; QList<QAction*> selhk; QStandardItemModel *im = nullptr; ImageItemDelegate *id = nullptr; - std::size_t ngroups, curgroup; + QProgressDialog *pd = nullptr; + SignatureDB *sdb = nullptr; + + std::size_t curgroup; bool nohotkeywarn; void mark_toggle(std::size_t x); void mark_all_but(std::size_t x); @@ -51,11 +59,17 @@ public: DeduperMainWindow(); void setup_menu(); - void show_images(const std::vector<std::filesystem::path> &fns); + void show_images(const std::vector<fs::path> &fns); void update_distances(const std::map<std::pair<std::size_t, std::size_t>, double> &d); void update_viewstatus(std::size_t cur, std::size_t size); void save_list(); void load_list(); + + void scan_dirs(std::vector<std::pair<fs::path, bool>> paths); +public Q_SLOTS: + void create_new(); + void update_actions(); + void show_group(size_t gid); Q_SIGNALS: void next(); void prev(); diff --git a/qdeduper/pathchooser.cpp b/qdeduper/pathchooser.cpp new file mode 100644 index 0000000..c08199f --- /dev/null +++ b/qdeduper/pathchooser.cpp @@ -0,0 +1,79 @@ +#include "pathchooser.hpp" + +#include <QDialogButtonBox> +#include <QLabel> +#include <QDebug> +#include <QFileDialog> +#include <QTableView> +#include <QPushButton> +#include <QStandardItemModel> +#include <QVBoxLayout> +#include <qdialogbuttonbox.h> +#include <qfiledialog.h> +#include <qnamespace.h> +#include <qstandarditemmodel.h> + +PathChooser::PathChooser(QWidget *parent) : QDialog(parent) +{ + QVBoxLayout *l = new QVBoxLayout(); + this->setLayout(l); + bb = new QDialogButtonBox(QDialogButtonBox::StandardButton::Ok | QDialogButtonBox::StandardButton::Cancel, this); + bb->button(QDialogButtonBox::StandardButton::Ok)->setText("Scan"); + + QPushButton *pbbrowse = new QPushButton(); + pbbrowse->setText("Browse..."); + pbbrowse->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DirOpenIcon)); + bb->addButton(pbbrowse, QDialogButtonBox::ButtonRole::ActionRole); + QObject::connect(pbbrowse, &QPushButton::pressed, this, &PathChooser::add_new); + + QPushButton *pbdelete = new QPushButton(); + pbdelete->setText("Delete"); + pbdelete->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DialogDiscardButton)); + bb->addButton(pbdelete, QDialogButtonBox::ButtonRole::ActionRole); + QObject::connect(pbdelete, &QPushButton::pressed, this, &PathChooser::delete_selected); + QObject::connect(bb, &QDialogButtonBox::accepted, this, &PathChooser::accept); + QObject::connect(bb, &QDialogButtonBox::rejected, this, &PathChooser::reject); + im = new QStandardItemModel(this); + tv = new QTableView(); + tv->setModel(im); + tv->setSortingEnabled(false); + tv->setSelectionMode(QAbstractItemView::SelectionMode::SingleSelection); + im->setHorizontalHeaderLabels({"Path", "Recursive?"}); + + QLabel *lb = new QLabel("Choose directories to scan"); + l->addWidget(lb); + l->addWidget(tv); + l->addWidget(bb); +} + +std::vector<std::pair<fs::path, bool>> PathChooser::get_dirs() +{ + std::vector<std::pair<fs::path, bool>> ret; + for (int i = 0; i < im->rowCount(); ++i) + { +#if PATH_VALSIZE == 2 + fs::path p(im->item(i, 0)->text().toStdWString()); +#else + fs::path p(im->item(i, 0)->text().toStdString()); +#endif + ret.emplace_back(p, (im->item(i, 1)->checkState() == Qt::CheckState::Checked)); + } + return ret; +} + +void PathChooser::add_new() +{ + QString s = QFileDialog::getExistingDirectory(this, "Open"); + if (s.isNull() || s.isEmpty()) return; + QStandardItem *it = new QStandardItem(s); + QStandardItem *ck = new QStandardItem(); + it->setEditable(false); + ck->setCheckable(true); + im->appendRow({it, ck}); + tv->resizeColumnsToContents(); +} + +void PathChooser::delete_selected() +{ + im->removeRow(tv->currentIndex().row()); +} diff --git a/qdeduper/pathchooser.hpp b/qdeduper/pathchooser.hpp new file mode 100644 index 0000000..07f9f51 --- /dev/null +++ b/qdeduper/pathchooser.hpp @@ -0,0 +1,31 @@ +#ifndef PATHCHOOSER_HPP +#define PATHCHOOSER_HPP + +#include <filesystem> +#include <utility> +#include <vector> + +#include <QDialog> + +namespace fs = std::filesystem; + +class QDialogButtonBox; +class QTableView; +class QStandardItemModel; + +class PathChooser : public QDialog +{ + Q_OBJECT +private: + QTableView *tv; + QStandardItemModel *im; + QDialogButtonBox *bb; +public: + PathChooser(QWidget *parent = nullptr); + std::vector<std::pair<fs::path, bool>> get_dirs(); +public Q_SLOTS: + void add_new(); + void delete_selected(); +}; + +#endif diff --git a/qdeduper/sigdb_qt.cpp b/qdeduper/sigdb_qt.cpp new file mode 100644 index 0000000..ab6a9f9 --- /dev/null +++ b/qdeduper/sigdb_qt.cpp @@ -0,0 +1,122 @@ +#include "sigdb_qt.hpp" +#include "signature_db.hpp" + +#include <algorithm> + +signature_config cfg_full = +{ + 9, //slices + 3, //blur_window + 2, //min_window + true, //crop + true, //comp + 0.5, //pr + 1./128, //noise_threshold + 0.05, //contrast_threshold + 0.25 //max_cropping +}; + +signature_config cfg_subslice = +{ + 4, //slices + 16, //blur_window + 2, //min_window + false, //crop + true, //comp + 0.5, //pr + 1./64, //noise_threshold + 0.05, //contrast_threshold + 0.25 //max_cropping +}; + +SignatureDB::SignatureDB() : QObject(nullptr) +{ + sdb = new signature_db(); +} + +SignatureDB::SignatureDB(const fs::path &dbpath) : QObject(nullptr) +{ + sdb = new signature_db(dbpath); +} + +SignatureDB::~SignatureDB() +{ + delete sdb; +} + +void SignatureDB::scan_files(const std::vector<fs::path> &files, int njobs) +{ + populate_cfg_t pcfg = { + 3, + 3, + cfg_full, + cfg_subslice, + 0.3, + [this](size_t c,int){Q_EMIT image_scanned(c);}, + njobs + }; + sdb->populate(files, pcfg); + + Q_EMIT image_scanned(~size_t(0)); + + auto ids = sdb->get_image_ids(); + sdb->batch_get_signature_begin(); + for (auto &id : ids) + { + fs::path p; + std::tie(p, std::ignore) = sdb->get_signature(id); + fmap[id] = p; + frmap[p] = id; + } + sdb->batch_get_signature_end(); + + auto dupes = sdb->dupe_pairs(); + for (auto &dupe : dupes) + distmap[std::make_pair(dupe.id1, dupe.id2)] = dupe.distance; + + sdb->group_similar(); + auto gps = sdb->groups_get(); + gps.erase(std::remove_if(gps.begin(), gps.end(), [](std::vector<size_t> v){ return v.size() < 2; }), gps.end()); + this->groups = std::move(gps); +} + +size_t SignatureDB::num_groups() +{ + return groups.size(); +} + +std::vector<size_t> SignatureDB::get_group(size_t gid) +{ + if (gid >= groups.size()) return {}; + return groups[gid]; +} + +std::map<std::pair<size_t, size_t>, double> SignatureDB::group_distances(size_t gid) +{ + std::map<std::pair<size_t, size_t>, double> ret; + auto g = get_group(gid); + for (size_t i = 0; i < g.size(); ++i) + for (size_t j = i + 1; j < g.size(); ++j) + { + size_t x = g[i], y = g[j]; + if (distmap.find(std::make_pair(x, y)) != distmap.end()) + ret[std::make_pair(i, j)] = distmap[std::make_pair(x, y)]; + else if (distmap.find(std::make_pair(y, x)) != distmap.end()) + ret[std::make_pair(j, i)] = distmap[std::make_pair(x, y)]; + } + return ret; +} + +fs::path SignatureDB::get_image_path(size_t id) +{ + if (fmap.find(id) == fmap.end()) + return fs::path(); + else return fmap[id]; +} + +size_t SignatureDB::get_path_id(const fs::path& p) +{ + if (frmap.find(p) == frmap.end()) + return ~size_t(0); + else return frmap[p]; +} diff --git a/qdeduper/sigdb_qt.hpp b/qdeduper/sigdb_qt.hpp new file mode 100644 index 0000000..8a5178a --- /dev/null +++ b/qdeduper/sigdb_qt.hpp @@ -0,0 +1,41 @@ +#ifndef SIGDB_QT_HPP +#define SIGDB_QT_HPP + +#include <filesystem> +#include <map> +#include <unordered_map> +#include <utility> +#include <vector> + +#include <QObject> + +#include "signature_db.hpp" + +namespace fs = std::filesystem; + +class SignatureDB : public QObject +{ + Q_OBJECT +private: + signature_db *sdb; + std::unordered_map<size_t, fs::path> fmap; + std::unordered_map<fs::path, size_t> frmap; + std::map<std::pair<size_t, size_t>, double> distmap; + std::vector<std::vector<size_t>> groups; +public: + SignatureDB(); + SignatureDB(const fs::path& dbpath); + ~SignatureDB(); + + void scan_files(const std::vector<fs::path> &files, int njobs); + size_t num_groups(); + std::vector<size_t> get_group(size_t gid); + std::map<std::pair<size_t, size_t>, double> group_distances(size_t gid); + + fs::path get_image_path(size_t id); + size_t get_path_id(const fs::path& p); +Q_SIGNALS: + void image_scanned(size_t n); +}; + +#endif diff --git a/xsig/include/signature_db.hpp b/xsig/include/signature_db.hpp index b37cf0a..107aa90 100644 --- a/xsig/include/signature_db.hpp +++ b/xsig/include/signature_db.hpp @@ -57,6 +57,8 @@ public: std::pair<fs::path, signature> get_signature(size_t id); void batch_get_signature_end(); + std::vector<size_t> get_image_ids(); + //place batch_put_subslice_begin() and batch_put_subslice_end() around a group of //put_subslice() calls to improve performance void batch_put_subslice_begin(); diff --git a/xsig/src/signature_db.cpp b/xsig/src/signature_db.cpp index 393b756..ba1a372 100644 --- a/xsig/src/signature_db.cpp +++ b/xsig/src/signature_db.cpp @@ -214,6 +214,22 @@ void signature_db::batch_get_signature_end() p->batch_end(batch_status::getsig); } +std::vector<size_t> signature_db::get_image_ids() +{ + sqlite3_stmt *st = nullptr; + sqlite3_prepare_v2(p->db, "select id from images;", -1, &st, 0); + std::vector<size_t> ret; + while (1) + { + int r = sqlite3_step(st); + if (r != SQLITE_ROW) break; + size_t id = (size_t)sqlite3_column_int(st, 0); + ret.push_back(id); + } + sqlite3_finalize(st); + return ret; +} + void signature_db::batch_put_subslice_begin() { if (!p->db) [[ unlikely ]] return; |