From 41e9051f2d809c42c3dfecc2eb11ad544cbd27b7 Mon Sep 17 00:00:00 2001 From: Chris Xiong Date: Mon, 19 Sep 2022 02:39:03 -0400 Subject: You break it, you fix it! The GUI is now working again, with scanning built-in. --- qdeduper/CMakeLists.txt | 6 +- qdeduper/filescanner.cpp | 75 ++++++++++++++++++++++ qdeduper/filescanner.hpp | 30 +++++++++ qdeduper/mingui.cpp | 162 +++++++++++++++++++++++++++++++++++++++-------- qdeduper/mingui.hpp | 18 +++++- qdeduper/pathchooser.cpp | 79 +++++++++++++++++++++++ qdeduper/pathchooser.hpp | 31 +++++++++ qdeduper/sigdb_qt.cpp | 122 +++++++++++++++++++++++++++++++++++ qdeduper/sigdb_qt.hpp | 41 ++++++++++++ 9 files changed, 534 insertions(+), 30 deletions(-) create mode 100644 qdeduper/filescanner.cpp create mode 100644 qdeduper/filescanner.hpp create mode 100644 qdeduper/pathchooser.cpp create mode 100644 qdeduper/pathchooser.hpp create mode 100644 qdeduper/sigdb_qt.cpp create mode 100644 qdeduper/sigdb_qt.hpp (limited to 'qdeduper') diff --git a/qdeduper/CMakeLists.txt b/qdeduper/CMakeLists.txt index f862c16..270cd45 100644 --- a/qdeduper/CMakeLists.txt +++ b/qdeduper/CMakeLists.txt @@ -1,6 +1,6 @@ set(CMAKE_INCLUDE_CURRENT_DIR ON) -find_package(Qt5 REQUIRED COMPONENTS Widgets) +find_package(Qt5 REQUIRED COMPONENTS Widgets Concurrent) set(CMAKE_AUTOMOC ON) set(CMAKE_AUTORCC ON) @@ -10,9 +10,13 @@ add_executable(qdeduper main.cpp mingui.cpp imageitem.cpp + sigdb_qt.cpp + filescanner.cpp + pathchooser.cpp ) target_link_libraries(qdeduper xsig Qt5::Widgets + Qt5::Concurrent ) diff --git a/qdeduper/filescanner.cpp b/qdeduper/filescanner.cpp new file mode 100644 index 0000000..e7e45fc --- /dev/null +++ b/qdeduper/filescanner.cpp @@ -0,0 +1,75 @@ +#include "filescanner.hpp" + +#include +#include +#include + +using std::size_t; + +FileScanner::FileScanner() : QObject(nullptr), maxmnlen(0) +{ + +} + +void FileScanner::add_magic_number(const std::string &m) +{ + mn.push_back(m); + if (m.length() > maxmnlen) + maxmnlen = m.length(); +} + +void FileScanner::add_path(const fs::path &p, bool recurse) +{ + paths.emplace_back(p, recurse); +} + +template +void dirit_foreach(T iter, std::function f) +{ + std::for_each(fs::begin(iter), fs::end(iter), f); +} + +void FileScanner::scan() +{ + size_t fcnt = 0; + auto opt = std::filesystem::directory_options::skip_permission_denied; + auto count_files = [&fcnt](const fs::directory_entry& e){ + if (e.is_regular_file()) ++fcnt; + }; + auto scan_file = [&fcnt, this](const fs::directory_entry &e) { + if (!e.is_regular_file()) return; + std::fstream fst(e.path(), std::ios::binary | std::ios::in); + std::string buf(maxmnlen, '\0'); + fst.read(buf.data(), maxmnlen); + buf.resize(fst.gcount()); + for (auto &magic : mn) + if (!memcmp(magic.data(), buf.data(), magic.length())) + { + ret.push_back(e.path()); + break; + } + Q_EMIT file_scanned(e.path(), ++fcnt); + }; + auto for_all_paths = [opt, this](std::function f) { + for (auto &pe : paths) + { + fs::path p; + bool recurse; + std::tie(p, recurse) = pe; + if (recurse) + dirit_foreach(fs::recursive_directory_iterator(p, opt), f); + else + dirit_foreach(fs::directory_iterator(p, opt), f); + } + }; + for_all_paths(count_files); + Q_EMIT scan_done_prep(fcnt); + + fcnt = 0; + for_all_paths(scan_file); +} + +std::vector FileScanner::file_list() +{ + return ret; +} diff --git a/qdeduper/filescanner.hpp b/qdeduper/filescanner.hpp new file mode 100644 index 0000000..5d927a4 --- /dev/null +++ b/qdeduper/filescanner.hpp @@ -0,0 +1,30 @@ +#ifndef FILESCANNER_HPP +#define FILESCANNER_HPP + +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +class FileScanner : public QObject +{ + Q_OBJECT + std::vector mn; + std::vector> paths; + std::vector ret; + std::size_t maxmnlen; +public: + FileScanner(); + void add_magic_number(const std::string &m); + void add_path(const fs::path &p, bool recurse = false); + void scan(); + std::vector file_list(); +Q_SIGNALS: + void scan_done_prep(std::size_t nfiles); + void file_scanned(const fs::path &p, std::size_t n); +}; + +#endif diff --git a/qdeduper/mingui.cpp b/qdeduper/mingui.cpp index 317076c..e936a36 100644 --- a/qdeduper/mingui.cpp +++ b/qdeduper/mingui.cpp @@ -1,11 +1,18 @@ #include "mingui.hpp" #include "imageitem.hpp" +#include "filescanner.hpp" +#include "pathchooser.hpp" +#include "sigdb_qt.hpp" #include +#include #include +#include #include #include +#include +#include #include #include #include @@ -16,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -83,6 +91,12 @@ DeduperMainWindow::DeduperMainWindow() lw->setHorizontalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel); lw->setVerticalScrollMode(QAbstractItemView::ScrollMode::ScrollPerPixel); lw->setMinimumWidth(240); + pd = new QProgressDialog(this); + pd->setModal(true); + pd->setMinimumDuration(0); + pd->setAutoReset(false); + pd->setAutoClose(false); + pd->close(); for (size_t i = 0; i < keys.size(); ++i) { @@ -105,29 +119,10 @@ DeduperMainWindow::DeduperMainWindow() mnone->setShortcut(QKeySequence(Qt::Key::Key_C)); QObject::connect(mnone, &QAction::triggered, [this]{this->mark_none();}); this->addAction(mnone); - QAction *nxt = new QAction(); - nxt->setShortcut(QKeySequence(Qt::Key::Key_M)); - QObject::connect(nxt, &QAction::triggered, [this]{Q_EMIT this->next();}); - this->addAction(nxt); - QAction *prv = new QAction(); - prv->setShortcut(QKeySequence(Qt::Key::Key_Z)); - QObject::connect(prv, &QAction::triggered, [this]{Q_EMIT this->prev();}); - this->addAction(prv); QAction *load = new QAction(); load->setShortcut(QKeySequence(Qt::Key::Key_N)); QObject::connect(load, &QAction::triggered, [this]{Q_EMIT this->load_list();}); this->addAction(load); - QAction *skip = new QAction(); - skip->setShortcut(QKeySequence(Qt::Key::Key_B)); - QObject::connect(skip, &QAction::triggered, [this]{ - bool ok = false; - int g = QInputDialog::getInt(this, "Skip to group", - QString("Group # (1-%1)").arg(ngroups), - curgroup + 1, - 1, ngroups, 1, &ok); - if (ok) Q_EMIT switch_group((size_t) g - 1); - }); - this->addAction(skip); QAction *save = new QAction(); save->setShortcut(QKeySequence(Qt::Modifier::SHIFT | Qt::Key::Key_Return)); QObject::connect(save, &QAction::triggered, [this]{Q_EMIT this->save_list();}); @@ -173,7 +168,9 @@ void DeduperMainWindow::setup_menu() QMenu *mark = this->menuBar()->addMenu("Marks"); QMenu *help = this->menuBar()->addMenu("Help"); - file->addAction("Create Database..."); + QAction *create_db = file->addAction("Create Database..."); + QObject::connect(create_db, &QAction::triggered, this, &DeduperMainWindow::create_new); + menuact["create_db"] = create_db; file->addAction("Load Database..."); file->addAction("Save Database..."); file->addSeparator(); @@ -182,8 +179,38 @@ void DeduperMainWindow::setup_menu() file->addAction("Preferences..."); file->addAction("Exit"); - view->addAction("Next Group"); - view->addAction("Previous Group"); + QAction *nxtgrp = view->addAction("Next Group"); + menuact["next_group"] = nxtgrp; + nxtgrp->setShortcut(QKeySequence(Qt::Key::Key_M)); + QObject::connect(nxtgrp, &QAction::triggered, [this] { + if (this->sdb && curgroup + 1 < this->sdb->num_groups()) + this->show_group(++curgroup); + }); + this->addAction(nxtgrp); + + QAction *prvgrp = view->addAction("Previous Group"); + menuact["prev_group"] = prvgrp; + prvgrp->setShortcut(QKeySequence(Qt::Key::Key_Z)); + QObject::connect(prvgrp, &QAction::triggered, [this] { + if (this->sdb && curgroup > 1) + this->show_group(--curgroup); + }); + this->addAction(prvgrp); + + QAction *skip = view->addAction("Skip to Group..."); + menuact["skip_group"] = skip; + skip->setShortcut(QKeySequence(Qt::Key::Key_B)); + QObject::connect(skip, &QAction::triggered, [this] { + if (!this->sdb) return; + bool ok = false; + int g = QInputDialog::getInt(this, "Skip to group", + QString("Group # (1-%1)").arg(sdb->num_groups()), + curgroup + 1, + 1, sdb->num_groups(), 1, &ok); + if (ok) this->show_group((size_t) g - 1); + }); + this->addAction(skip); + view->addSeparator(); QMenu *sort = view->addMenu("Sort by"); sort->addAction("File size"); @@ -198,16 +225,21 @@ void DeduperMainWindow::setup_menu() help->addAction("View Documentation"); help->addAction("About"); } +void DeduperMainWindow::update_actions() +{ + if (!sdb) + { + menuact["next_group"]->setEnabled(false); + menuact["prev_group"]->setEnabled(false); + return; + } +} void DeduperMainWindow::show_images(const std::vector &fns) { current_set = fns; marks.clear(); im->clear(); - int max_height = (this->screen()->size().height() / fns.size() * 0.8 - 24) * this->screen()->devicePixelRatio(); - int max_width = this->screen()->size().width() * 0.8 * this->screen()->devicePixelRatio(); - if (max_height < 64) max_height = 64; - if (max_width < 64) max_width = 64; fs::path::string_type common_pfx = common_prefix(fns); size_t idx = 0; if (fns.size() > keys.size() && !nohotkeywarn) @@ -245,7 +277,6 @@ void DeduperMainWindow::update_distances(const std::mapsetText(QString("Viewing group %1 of %2").arg(cur + 1).arg(size)); - ngroups = size; curgroup = cur; } @@ -291,6 +322,83 @@ void DeduperMainWindow::load_list() mark_view_update(); } +void DeduperMainWindow::create_new() +{ + PathChooser *pc = new PathChooser(this); + pc->setModal(true); + if (pc->exec() == QDialog::DialogCode::Accepted) + this->scan_dirs(pc->get_dirs()); + pc->deleteLater(); +} + +void DeduperMainWindow::scan_dirs(std::vector> paths) +{ + this->pd->open(); + this->pd->setLabelText("Preparing for database creation..."); + this->pd->setMinimum(0); + this->pd->setMaximum(0); + auto f = QtConcurrent::run([this, paths] { + FileScanner *fs = new FileScanner(); + std::for_each(paths.begin(), paths.end(), [fs](auto p){fs->add_path(p.first, p.second);}); + fs->add_magic_number("\x89PNG\r\n"); + fs->add_magic_number("\xff\xd8\xff"); + QObject::connect(fs, &FileScanner::scan_done_prep, [this](auto n) { + this->pd->setMaximum(n - 1); + }); + QObject::connect(fs, &FileScanner::file_scanned, [this](const fs::path &p, size_t c) { + static auto lt = std::chrono::steady_clock::now(); + using namespace std::literals; + if (std::chrono::steady_clock::now() - lt > 100ms) + { + lt = std::chrono::steady_clock::now(); + this->pd->setLabelText(QString("Looking for files to scan: %1").arg(fsstr_to_qstring(p))); + this->pd->setValue(c); + } + }); + fs->scan(); + this->pd->setMaximum(fs->file_list().size() - 1); + this->pd->setLabelText("Scanning..."); + this->sdb = new SignatureDB(); + QObject::connect(this->sdb, &SignatureDB::image_scanned, this, [this](size_t n) { + static auto lt = std::chrono::steady_clock::now(); + using namespace std::literals; + if (std::chrono::steady_clock::now() - lt > 100ms) + { + lt = std::chrono::steady_clock::now(); + this->pd->setLabelText(QString("Scanning %1 / %2").arg(n + 1).arg(this->pd->maximum() + 1)); + this->pd->setValue(n); + } + if (!~n) + { + this->pd->setMaximum(0); + this->pd->setLabelText("Finalizing..."); + } + }, Qt::ConnectionType::QueuedConnection); + this->sdb->scan_files(fs->file_list(), 8); + delete fs; + }); + QFutureWatcher *fw = new QFutureWatcher(this); + fw->setFuture(f); + QObject::connect(fw, &QFutureWatcher::finished, this, [this] { + this->pd->reset(); + this->pd->close(); + this->curgroup = 0; + this->show_group(this->curgroup); + }, Qt::ConnectionType::QueuedConnection); +} + +void DeduperMainWindow::show_group(size_t gid) +{ + if (!sdb || gid >= sdb->num_groups()) + return; + auto g = sdb->get_group(gid); + current_set.clear(); + std::for_each(g.begin(), g.end(), [this](auto id){current_set.push_back(sdb->get_image_path(id));}); + this->show_images(current_set); + this->update_distances(sdb->group_distances(gid)); + this->update_viewstatus(gid, sdb->num_groups()); +} + void DeduperMainWindow::mark_toggle(size_t x) { if (x < marks.size()) diff --git a/qdeduper/mingui.hpp b/qdeduper/mingui.hpp index b8d13ef..3515047 100644 --- a/qdeduper/mingui.hpp +++ b/qdeduper/mingui.hpp @@ -2,6 +2,7 @@ #define MINGUI_HPP #include +#include #include #include #include @@ -9,12 +10,15 @@ #include #include +#include "sigdb_qt.hpp" + class QHBoxLayout; class QLabel; class QStatusBar; class QScrollArea; class QTextEdit; class QListView; +class QProgressDialog; class QSplitter; class QStandardItemModel; class ImageItemDelegate; @@ -30,10 +34,14 @@ private: QLabel *permamsg; QStatusBar *sb; QListView *lw; + std::map menuact; QList selhk; QStandardItemModel *im = nullptr; ImageItemDelegate *id = nullptr; - std::size_t ngroups, curgroup; + QProgressDialog *pd = nullptr; + SignatureDB *sdb = nullptr; + + std::size_t curgroup; bool nohotkeywarn; void mark_toggle(std::size_t x); void mark_all_but(std::size_t x); @@ -51,11 +59,17 @@ public: DeduperMainWindow(); void setup_menu(); - void show_images(const std::vector &fns); + void show_images(const std::vector &fns); void update_distances(const std::map, double> &d); void update_viewstatus(std::size_t cur, std::size_t size); void save_list(); void load_list(); + + void scan_dirs(std::vector> paths); +public Q_SLOTS: + void create_new(); + void update_actions(); + void show_group(size_t gid); Q_SIGNALS: void next(); void prev(); diff --git a/qdeduper/pathchooser.cpp b/qdeduper/pathchooser.cpp new file mode 100644 index 0000000..c08199f --- /dev/null +++ b/qdeduper/pathchooser.cpp @@ -0,0 +1,79 @@ +#include "pathchooser.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +PathChooser::PathChooser(QWidget *parent) : QDialog(parent) +{ + QVBoxLayout *l = new QVBoxLayout(); + this->setLayout(l); + bb = new QDialogButtonBox(QDialogButtonBox::StandardButton::Ok | QDialogButtonBox::StandardButton::Cancel, this); + bb->button(QDialogButtonBox::StandardButton::Ok)->setText("Scan"); + + QPushButton *pbbrowse = new QPushButton(); + pbbrowse->setText("Browse..."); + pbbrowse->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DirOpenIcon)); + bb->addButton(pbbrowse, QDialogButtonBox::ButtonRole::ActionRole); + QObject::connect(pbbrowse, &QPushButton::pressed, this, &PathChooser::add_new); + + QPushButton *pbdelete = new QPushButton(); + pbdelete->setText("Delete"); + pbdelete->setIcon(this->style()->standardIcon(QStyle::StandardPixmap::SP_DialogDiscardButton)); + bb->addButton(pbdelete, QDialogButtonBox::ButtonRole::ActionRole); + QObject::connect(pbdelete, &QPushButton::pressed, this, &PathChooser::delete_selected); + QObject::connect(bb, &QDialogButtonBox::accepted, this, &PathChooser::accept); + QObject::connect(bb, &QDialogButtonBox::rejected, this, &PathChooser::reject); + im = new QStandardItemModel(this); + tv = new QTableView(); + tv->setModel(im); + tv->setSortingEnabled(false); + tv->setSelectionMode(QAbstractItemView::SelectionMode::SingleSelection); + im->setHorizontalHeaderLabels({"Path", "Recursive?"}); + + QLabel *lb = new QLabel("Choose directories to scan"); + l->addWidget(lb); + l->addWidget(tv); + l->addWidget(bb); +} + +std::vector> PathChooser::get_dirs() +{ + std::vector> ret; + for (int i = 0; i < im->rowCount(); ++i) + { +#if PATH_VALSIZE == 2 + fs::path p(im->item(i, 0)->text().toStdWString()); +#else + fs::path p(im->item(i, 0)->text().toStdString()); +#endif + ret.emplace_back(p, (im->item(i, 1)->checkState() == Qt::CheckState::Checked)); + } + return ret; +} + +void PathChooser::add_new() +{ + QString s = QFileDialog::getExistingDirectory(this, "Open"); + if (s.isNull() || s.isEmpty()) return; + QStandardItem *it = new QStandardItem(s); + QStandardItem *ck = new QStandardItem(); + it->setEditable(false); + ck->setCheckable(true); + im->appendRow({it, ck}); + tv->resizeColumnsToContents(); +} + +void PathChooser::delete_selected() +{ + im->removeRow(tv->currentIndex().row()); +} diff --git a/qdeduper/pathchooser.hpp b/qdeduper/pathchooser.hpp new file mode 100644 index 0000000..07f9f51 --- /dev/null +++ b/qdeduper/pathchooser.hpp @@ -0,0 +1,31 @@ +#ifndef PATHCHOOSER_HPP +#define PATHCHOOSER_HPP + +#include +#include +#include + +#include + +namespace fs = std::filesystem; + +class QDialogButtonBox; +class QTableView; +class QStandardItemModel; + +class PathChooser : public QDialog +{ + Q_OBJECT +private: + QTableView *tv; + QStandardItemModel *im; + QDialogButtonBox *bb; +public: + PathChooser(QWidget *parent = nullptr); + std::vector> get_dirs(); +public Q_SLOTS: + void add_new(); + void delete_selected(); +}; + +#endif diff --git a/qdeduper/sigdb_qt.cpp b/qdeduper/sigdb_qt.cpp new file mode 100644 index 0000000..ab6a9f9 --- /dev/null +++ b/qdeduper/sigdb_qt.cpp @@ -0,0 +1,122 @@ +#include "sigdb_qt.hpp" +#include "signature_db.hpp" + +#include + +signature_config cfg_full = +{ + 9, //slices + 3, //blur_window + 2, //min_window + true, //crop + true, //comp + 0.5, //pr + 1./128, //noise_threshold + 0.05, //contrast_threshold + 0.25 //max_cropping +}; + +signature_config cfg_subslice = +{ + 4, //slices + 16, //blur_window + 2, //min_window + false, //crop + true, //comp + 0.5, //pr + 1./64, //noise_threshold + 0.05, //contrast_threshold + 0.25 //max_cropping +}; + +SignatureDB::SignatureDB() : QObject(nullptr) +{ + sdb = new signature_db(); +} + +SignatureDB::SignatureDB(const fs::path &dbpath) : QObject(nullptr) +{ + sdb = new signature_db(dbpath); +} + +SignatureDB::~SignatureDB() +{ + delete sdb; +} + +void SignatureDB::scan_files(const std::vector &files, int njobs) +{ + populate_cfg_t pcfg = { + 3, + 3, + cfg_full, + cfg_subslice, + 0.3, + [this](size_t c,int){Q_EMIT image_scanned(c);}, + njobs + }; + sdb->populate(files, pcfg); + + Q_EMIT image_scanned(~size_t(0)); + + auto ids = sdb->get_image_ids(); + sdb->batch_get_signature_begin(); + for (auto &id : ids) + { + fs::path p; + std::tie(p, std::ignore) = sdb->get_signature(id); + fmap[id] = p; + frmap[p] = id; + } + sdb->batch_get_signature_end(); + + auto dupes = sdb->dupe_pairs(); + for (auto &dupe : dupes) + distmap[std::make_pair(dupe.id1, dupe.id2)] = dupe.distance; + + sdb->group_similar(); + auto gps = sdb->groups_get(); + gps.erase(std::remove_if(gps.begin(), gps.end(), [](std::vector v){ return v.size() < 2; }), gps.end()); + this->groups = std::move(gps); +} + +size_t SignatureDB::num_groups() +{ + return groups.size(); +} + +std::vector SignatureDB::get_group(size_t gid) +{ + if (gid >= groups.size()) return {}; + return groups[gid]; +} + +std::map, double> SignatureDB::group_distances(size_t gid) +{ + std::map, double> ret; + auto g = get_group(gid); + for (size_t i = 0; i < g.size(); ++i) + for (size_t j = i + 1; j < g.size(); ++j) + { + size_t x = g[i], y = g[j]; + if (distmap.find(std::make_pair(x, y)) != distmap.end()) + ret[std::make_pair(i, j)] = distmap[std::make_pair(x, y)]; + else if (distmap.find(std::make_pair(y, x)) != distmap.end()) + ret[std::make_pair(j, i)] = distmap[std::make_pair(x, y)]; + } + return ret; +} + +fs::path SignatureDB::get_image_path(size_t id) +{ + if (fmap.find(id) == fmap.end()) + return fs::path(); + else return fmap[id]; +} + +size_t SignatureDB::get_path_id(const fs::path& p) +{ + if (frmap.find(p) == frmap.end()) + return ~size_t(0); + else return frmap[p]; +} diff --git a/qdeduper/sigdb_qt.hpp b/qdeduper/sigdb_qt.hpp new file mode 100644 index 0000000..8a5178a --- /dev/null +++ b/qdeduper/sigdb_qt.hpp @@ -0,0 +1,41 @@ +#ifndef SIGDB_QT_HPP +#define SIGDB_QT_HPP + +#include +#include +#include +#include +#include + +#include + +#include "signature_db.hpp" + +namespace fs = std::filesystem; + +class SignatureDB : public QObject +{ + Q_OBJECT +private: + signature_db *sdb; + std::unordered_map fmap; + std::unordered_map frmap; + std::map, double> distmap; + std::vector> groups; +public: + SignatureDB(); + SignatureDB(const fs::path& dbpath); + ~SignatureDB(); + + void scan_files(const std::vector &files, int njobs); + size_t num_groups(); + std::vector get_group(size_t gid); + std::map, double> group_distances(size_t gid); + + fs::path get_image_path(size_t id); + size_t get_path_id(const fs::path& p); +Q_SIGNALS: + void image_scanned(size_t n); +}; + +#endif -- cgit v1.2.3