Add image verification

"Verify Dump" is now removed, now both hash calculation
and image verification are done in one step.
After a successful hash calculation, the button is replaced with
a "Search on Redump.org" button that opens a web browser
on Redump's search page.

src/common/cd_image_hasher.cpp

@@ -82,6 +82,17 @@ std::string HashToString(const Hash& hash)
                                          hash[9], hash[10], hash[11], hash[12], hash[13], hash[14], hash[15]);
 }
 
+std::optional<Hash> HashFromString(const std::string_view& str) {
+    auto decoded = StringUtil::DecodeHex(str);
+    if (decoded && decoded->size() == std::tuple_size_v<Hash>)
+    {
+        Hash result;
+        std::copy(decoded->begin(), decoded->end(), result.begin());
+        return result;
+    }
+    return std::nullopt;
+}
+
 bool GetImageHash(CDImage* image, Hash* out_hash,
                   ProgressCallback* progress_callback /*= ProgressCallback::NullProgressCallback*/)
 {

src/common/cd_image_hasher.h

@@ -2,6 +2,7 @@
 #include "progress_callback.h"
 #include "types.h"
 #include <array>
+#include <optional>
 #include <string>
 
 class CDImage;
@@ -10,6 +11,7 @@ namespace CDImageHasher {
 
 using Hash = std::array<u8, 16>;
 std::string HashToString(const Hash& hash);
+std::optional<Hash> HashFromString(const std::string_view& str);
 
 bool GetImageHash(CDImage* image, Hash* out_hash,
                   ProgressCallback* progress_callback = ProgressCallback::NullProgressCallback);

src/duckstation-qt/gamepropertiesdialog.cpp

@@ -1,18 +1,23 @@
 #include "gamepropertiesdialog.h"
 #include "common/cd_image.h"
 #include "common/cd_image_hasher.h"
+#include "common/string_util.h"
 #include "core/settings.h"
 #include "core/system.h"
+#include "frontend-common/game_database.h"
 #include "frontend-common/game_list.h"
 #include "qthostinterface.h"
 #include "qtprogresscallback.h"
 #include "qtutils.h"
+#include "rapidjson/document.h"
 #include "scmversion/scmversion.h"
 #include <QtGui/QClipboard>
 #include <QtGui/QGuiApplication>
 #include <QtWidgets/QFileDialog>
 #include <QtWidgets/QInputDialog>
 #include <QtWidgets/QMessageBox>
+#include <map>
+Log_SetChannel(GamePropertiesDialog);
 
 static constexpr char MEMORY_CARD_IMAGE_FILTER[] =
   QT_TRANSLATE_NOOP("MemoryCardSettingsWidget", "All Memory Card Types (*.mcd *.mcr *.mc)");
@@ -71,6 +76,7 @@ void GamePropertiesDialog::populate(const GameListEntry* ge)
     m_ui.gameCode->setText(QStringLiteral("%1 / %2").arg(ge->code.c_str()).arg(hash_code.c_str()));
   else
     m_ui.gameCode->setText(QString::fromStdString(ge->code));
+  m_ui.revision->setText(tr("<not verified>"));
 
   m_ui.region->setCurrentIndex(static_cast<int>(ge->region));
 
@@ -83,7 +89,6 @@ void GamePropertiesDialog::populate(const GameListEntry* ge)
     m_ui.comments->setDisabled(true);
     m_ui.versionTested->setDisabled(true);
     m_ui.setToCurrent->setDisabled(true);
-    m_ui.verifyDump->setDisabled(true);
     m_ui.exportCompatibilityInfo->setDisabled(true);
   }
   else
@@ -261,6 +266,10 @@ void GamePropertiesDialog::populateTracksInfo(const std::string& image_path)
     m_ui.tracks->setItem(row, 2, new QTableWidgetItem(MSFTotString(position)));
     m_ui.tracks->setItem(row, 3, new QTableWidgetItem(MSFTotString(length)));
     m_ui.tracks->setItem(row, 4, new QTableWidgetItem(tr("<not computed>")));
+
+    QTableWidgetItem* status = new QTableWidgetItem(QString());
+    status->setTextAlignment(Qt::AlignCenter);
+    m_ui.tracks->setItem(row, 5, status);
   }
 }
 
@@ -532,7 +541,7 @@ void GamePropertiesDialog::resizeEvent(QResizeEvent* ev)
 
 void GamePropertiesDialog::onResize()
 {
-  QtUtils::ResizeColumnsForTableView(m_ui.tracks, {20, 85, 125, 125, -1});
+  QtUtils::ResizeColumnsForTableView(m_ui.tracks, {15, 85, 125, 125, -1, 25});
 }
 
 void GamePropertiesDialog::connectUi()
@@ -546,14 +555,12 @@ void GamePropertiesDialog::connectUi()
           &GamePropertiesDialog::saveCompatibilityInfoIfChanged);
   connect(m_ui.setToCurrent, &QPushButton::clicked, this, &GamePropertiesDialog::onSetVersionTestedToCurrentClicked);
   connect(m_ui.computeHashes, &QPushButton::clicked, this, &GamePropertiesDialog::onComputeHashClicked);
-  connect(m_ui.verifyDump, &QPushButton::clicked, this, &GamePropertiesDialog::onVerifyDumpClicked);
   connect(m_ui.exportCompatibilityInfo, &QPushButton::clicked, this,
           &GamePropertiesDialog::onExportCompatibilityInfoClicked);
   connect(m_ui.close, &QPushButton::clicked, this, &QDialog::close);
   connect(m_ui.tabWidget, &QTabWidget::currentChanged, [this](int index) {
     const bool show_buttons = index == 0;
     m_ui.computeHashes->setVisible(show_buttons);
-    m_ui.verifyDump->setVisible(show_buttons);
     m_ui.exportCompatibilityInfo->setVisible(show_buttons);
   });
 
@@ -924,15 +931,19 @@ void GamePropertiesDialog::onSetVersionTestedToCurrentClicked()
 
 void GamePropertiesDialog::onComputeHashClicked()
 {
-  if (m_tracks_hashed)
-    return;
-
-  computeTrackHashes();
-}
+  if (m_redump_search_keyword.empty())
+  {
+    computeTrackHashes(m_redump_search_keyword);
 
-void GamePropertiesDialog::onVerifyDumpClicked()
-{
-  QMessageBox::critical(this, tr("Not yet implemented"), tr("Not yet implemented"));
+    if (!m_redump_search_keyword.empty())
+      m_ui.computeHashes->setText(tr("Search on Redump.org"));
+  }
+  else
+  {
+    QtUtils::OpenURL(
+      this, StringUtil::StdStringFromFormat("http://redump.org/discs/quicksearch/%s", m_redump_search_keyword.c_str())
+              .c_str());
+  }
 }
 
 void GamePropertiesDialog::onExportCompatibilityInfoClicked()
@@ -952,7 +963,7 @@ void GamePropertiesDialog::onExportCompatibilityInfoClicked()
     QGuiApplication::clipboard()->setText(xml);
 }
 
-void GamePropertiesDialog::computeTrackHashes()
+void GamePropertiesDialog::computeTrackHashes(std::string& redump_keyword)
 {
   if (m_path.empty())
     return;
@@ -961,9 +972,25 @@ void GamePropertiesDialog::computeTrackHashes()
   if (!image)
     return;
 
+  // Kick off hash preparation asynchronously, as building the map of results may take a while
+  auto hashes_map_job = std::async(std::launch::async, [] {
+    GameDatabase::TrackHashesMap result;
+    GameDatabase db;
+    if (db.Load())
+    {
+      result = db.GetTrackHashesMap();
+    }
+    return result;
+  });
+
   QtProgressCallback progress_callback(this);
   progress_callback.SetProgressRange(image->GetTrackCount());
 
+  std::vector<CDImageHasher::Hash> track_hashes;
+  track_hashes.reserve(image->GetTrackCount());
+
+  // Calculate hashes
+  bool calculate_hash_success = true;
   for (u8 track = 1; track <= image->GetTrackCount(); track++)
   {
     progress_callback.SetProgressValue(track - 1);
@@ -973,14 +1000,98 @@ void GamePropertiesDialog::computeTrackHashes()
     if (!CDImageHasher::GetTrackHash(image.get(), track, &hash, &progress_callback))
     {
       progress_callback.PopState();
+      calculate_hash_success = false;
       break;
     }
-
-    QString hash_string(QString::fromStdString(CDImageHasher::HashToString(hash)));
+    track_hashes.emplace_back(hash);
 
     QTableWidgetItem* item = m_ui.tracks->item(track - 1, 4);
-    item->setText(hash_string);
+    item->setText(QString::fromStdString(CDImageHasher::HashToString(hash)));
 
     progress_callback.PopState();
   }
+
+  // Verify hashes against gamedb
+  std::vector<bool> verification_results(image->GetTrackCount(), false);
+  if (calculate_hash_success)
+  {
+    std::string found_revision;
+    redump_keyword = CDImageHasher::HashToString(track_hashes.front());
+
+    progress_callback.SetStatusText("Verifying hashes...");
+    progress_callback.SetProgressValue(image->GetTrackCount());
+
+    const auto hashes_map = hashes_map_job.get();
+
+    // Verification strategy used:
+    // 1. First, find all matches for the data track
+    //    If none are found, fail verification for all tracks
+    // 2. For each data track match, try to match all audio tracks
+    //    If all match, assume this revision. Else, try other revisions,
+    //    and accept the one with the most matches.
+    auto data_track_matches = hashes_map.equal_range(track_hashes[0]);
+    if (data_track_matches.first != data_track_matches.second)
+    {
+      auto best_data_match = data_track_matches.second;
+      for (auto iter = data_track_matches.first; iter != data_track_matches.second; ++iter)
+      {
+        std::vector<bool> current_verification_results(image->GetTrackCount(), false);
+        const auto& data_track_attribs = iter->second;
+        current_verification_results[0] = true; // Data track already matched
+
+        for (auto audio_tracks_iter = std::next(track_hashes.begin()); audio_tracks_iter != track_hashes.end();
+             ++audio_tracks_iter)
+        {
+          auto audio_track_matches = hashes_map.equal_range(*audio_tracks_iter);
+          for (auto audio_iter = audio_track_matches.first; audio_iter != audio_track_matches.second; ++audio_iter)
+          {
+            // If audio track comes from the same revision and code as the data track, "pass" it
+            if (audio_iter->second == data_track_attribs)
+            {
+              current_verification_results[std::distance(track_hashes.begin(), audio_tracks_iter)] = true;
+              break;
+            }
+          }
+        }
+
+        const auto old_matches_count = std::count(verification_results.begin(), verification_results.end(), true);
+        const auto new_matches_count =
+          std::count(current_verification_results.begin(), current_verification_results.end(), true);
+
+        if (new_matches_count > old_matches_count)
+        {
+          best_data_match = iter;
+          verification_results = current_verification_results;
+          // If all elements got matched, early out
+          if (new_matches_count >= static_cast<ptrdiff_t>(verification_results.size()))
+          {
+            break;
+          }
+        }
+      }
+
+      found_revision = best_data_match->second.revisionString;
+    }
+
+    m_ui.revision->setText(!found_revision.empty() ? QString::fromStdString(found_revision) : QStringLiteral("-"));
+  }
+
+  for (u8 track = 0; track < image->GetTrackCount(); track++)
+  {
+    QTableWidgetItem* hash_text = m_ui.tracks->item(track, 4);
+    QTableWidgetItem* status_text = m_ui.tracks->item(track, 5);
+    QBrush brush;
+    if (verification_results[track])
+    {
+      brush = QColor(0, 200, 0);
+      status_text->setText(QString::fromUtf8(u8"\u2713"));
+    }
+    else
+    {
+      brush = QColor(200, 0, 0);
+      status_text->setText(QString::fromUtf8(u8"\u2715"));
+    }
+    status_text->setForeground(brush);
+    hash_text->setForeground(brush);
+  }
 }

src/duckstation-qt/gamepropertiesdialog.h

@@ -34,7 +34,6 @@ private Q_SLOTS:
 
   void onSetVersionTestedToCurrentClicked();
   void onComputeHashClicked();
-  void onVerifyDumpClicked();
   void onExportCompatibilityInfoClicked();
   void updateCPUClockSpeedLabel();
   void onEnableCPUClockSpeedControlChecked(int state);
@@ -49,7 +48,7 @@ private:
   void connectBooleanUserSetting(QCheckBox* cb, std::optional<bool>* value);
   void saveGameSettings();
   void fillEntryFromUi(GameListCompatibilityEntry* entry);
-  void computeTrackHashes();
+  void computeTrackHashes(std::string& redump_keyword);
   void onResize();
   void onUserAspectRatioChanged();
 
@@ -61,9 +60,9 @@ private:
   std::string m_path;
   std::string m_game_code;
   std::string m_game_title;
+  std::string m_redump_search_keyword;
 
   GameSettings::Entry m_game_settings;
 
   bool m_compatibility_info_changed = false;
-  bool m_tracks_hashed = false;
 };

src/duckstation-qt/gamepropertiesdialog.ui

@@ -70,58 +70,58 @@
          </property>
         </widget>
        </item>
-       <item row="3" column="0">
+       <item row="4" column="0">
         <widget class="QLabel" name="label_3">
          <property name="text">
           <string>Region:</string>
          </property>
         </widget>
        </item>
-       <item row="3" column="1">
+       <item row="4" column="1">
         <widget class="QComboBox" name="region">
          <property name="enabled">
           <bool>false</bool>
          </property>
         </widget>
        </item>
-       <item row="4" column="0">
+       <item row="5" column="0">
         <widget class="QLabel" name="label_7">
          <property name="text">
           <string>Compatibility:</string>
          </property>
         </widget>
        </item>
-       <item row="4" column="1">
+       <item row="5" column="1">
         <widget class="QComboBox" name="compatibility"/>
        </item>
-       <item row="5" column="0">
+       <item row="6" column="0">
         <widget class="QLabel" name="label_6">
          <property name="text">
           <string>Upscaling Issues:</string>
          </property>
         </widget>
        </item>
-       <item row="5" column="1">
+       <item row="6" column="1">
         <widget class="QLineEdit" name="upscalingIssues"/>
        </item>
-       <item row="6" column="0">
+       <item row="7" column="0">
         <widget class="QLabel" name="label_5">
          <property name="text">
           <string>Comments:</string>
          </property>
         </widget>
        </item>
-       <item row="6" column="1">
+       <item row="7" column="1">
         <widget class="QLineEdit" name="comments"/>
        </item>
-       <item row="7" column="0">
+       <item row="8" column="0">
         <widget class="QLabel" name="label_9">
          <property name="text">
           <string>Version Tested:</string>
          </property>
         </widget>
        </item>
-       <item row="7" column="1">
+       <item row="8" column="1">
         <layout class="QHBoxLayout" name="horizontalLayout_3">
          <item>
           <widget class="QLineEdit" name="versionTested"/>
@@ -135,14 +135,14 @@
          </item>
         </layout>
        </item>
-       <item row="8" column="0" colspan="2">
+       <item row="9" column="0" colspan="2">
         <widget class="QLabel" name="label_8">
          <property name="text">
           <string>Tracks:</string>
          </property>
         </widget>
        </item>
-       <item row="9" column="0" colspan="2">
+       <item row="10" column="0" colspan="2">
         <widget class="QTableWidget" name="tracks">
          <property name="editTriggers">
           <set>QAbstractItemView::NoEditTriggers</set>
@@ -178,6 +178,28 @@
            <string>Hash</string>
           </property>
          </column>
+         <column>
+          <property name="text">
+           <string>Status</string>
+          </property>
+         </column>
+        </widget>
+       </item>
+       <item row="3" column="1">
+        <widget class="QLineEdit" name="revision">
+         <property name="readOnly">
+          <bool>true</bool>
+         </property>
+         <property name="placeholderText">
+          <string/>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="0">
+        <widget class="QLabel" name="label_36">
+         <property name="text">
+          <string>Revision:</string>
+         </property>
         </widget>
        </item>
       </layout>
@@ -258,7 +280,7 @@
              <string>Enable 8MB RAM (Dev Console)</string>
             </property>
             <property name="tristate">
-              <bool>true</bool>
+             <bool>true</bool>
             </property>
            </widget>
           </item>
@@ -1119,14 +1141,7 @@
      <item>
       <widget class="QPushButton" name="computeHashes">
        <property name="text">
-        <string>Compute Hashes</string>
-       </property>
-      </widget>
-     </item>
-     <item>
-      <widget class="QPushButton" name="verifyDump">
-       <property name="text">
-        <string>Verify Dump</string>
+        <string>Compute &amp;&amp; Verify Hashes</string>
        </property>
       </widget>
      </item>

src/duckstation-qt/qtprogresscallback.cpp

@@ -10,6 +10,8 @@ QtProgressCallback::QtProgressCallback(QWidget* parent_widget, float show_delay)
   m_dialog.setWindowTitle(tr("DuckStation"));
   m_dialog.setMinimumSize(QSize(500, 0));
   m_dialog.setModal(parent_widget != nullptr);
+  m_dialog.setAutoClose(false);
+  m_dialog.setAutoReset(false);
   checkForDelayedShow();
 }
 
@@ -57,10 +59,9 @@ void QtProgressCallback::SetProgressValue(u32 value)
   BaseProgressCallback::SetProgressValue(value);
   checkForDelayedShow();
 
-  if (!m_dialog.isVisible() || static_cast<u32>(m_dialog.value()) == m_progress_range)
-    return;
+  if (m_dialog.isVisible() && static_cast<u32>(m_dialog.value()) != m_progress_range)
+    m_dialog.setValue(m_progress_value);
 
-  m_dialog.setValue(m_progress_value);
   QCoreApplication::processEvents();
 }
 

src/frontend-common/game_database.cpp

@@ -8,6 +8,8 @@
 #include "rapidjson/document.h"
 #include "rapidjson/error/en.h"
 #include <iomanip>
+#include <memory>
+#include <optional>
 #include <sstream>
 Log_SetChannel(GameDatabase);
 
@@ -87,6 +89,23 @@ static bool GetUIntFromObject(const rapidjson::Value& object, const char* key, u
   return true;
 }
 
+static bool GetArrayOfStringsFromObject(const rapidjson::Value& object, const char* key, std::vector<std::string>* dest)
+{
+  dest->clear();
+  auto member = object.FindMember(key);
+  if (member == object.MemberEnd() || !member->value.IsArray())
+    return false;
+
+  for (const rapidjson::Value& str : member->value.GetArray())
+  {
+    if (str.IsString())
+    {
+      dest->emplace_back(str.GetString(), str.GetStringLength());
+    }
+  }
+  return true;
+}
+
 static const rapidjson::Value* FindDatabaseEntry(const std::string_view& code, rapidjson::Document* json)
 {
   for (const rapidjson::Value& current : json->GetArray())
@@ -129,7 +148,7 @@ static const rapidjson::Value* FindDatabaseEntry(const std::string_view& code, r
   return nullptr;
 }
 
-bool GameDatabase::GetEntryForCode(const std::string_view& code, GameDatabaseEntry* entry)
+bool GameDatabase::GetEntryForCode(const std::string_view& code, GameDatabaseEntry* entry) const
 {
   if (!m_json)
     return false;
@@ -213,7 +232,88 @@ bool GameDatabase::GetEntryForCode(const std::string_view& code, GameDatabaseEnt
   return true;
 }
 
-bool GameDatabase::GetEntryForDisc(CDImage* image, GameDatabaseEntry* entry)
+GameDatabase::TrackHashesMap GameDatabase::GetTrackHashesMap() const
+{
+  TrackHashesMap result;
+
+  auto json = static_cast<const rapidjson::Document*>(m_json);
+
+  for (const rapidjson::Value& current : json->GetArray())
+  {
+    if (!current.IsObject())
+    {
+      Log_WarningPrintf("entry is not an object");
+      continue;
+    }
+
+    std::vector<std::string> codes;
+    if (!GetArrayOfStringsFromObject(current, "codes", &codes))
+    {
+      Log_WarningPrintf("codes member is missing");
+      continue;
+    }
+
+    auto track_data = current.FindMember("track_data");
+    if (track_data == current.MemberEnd())
+    {
+      Log_WarningPrintf("track_data member is missing");
+      continue;
+    }
+
+    if (!track_data->value.IsArray())
+    {
+      Log_WarningPrintf("track_data is not an array");
+      continue;
+    }
+
+    uint32_t revision = 0;
+    for (const rapidjson::Value& track_revisions : track_data->value.GetArray())
+    {
+      if (!track_revisions.IsObject())
+      {
+        Log_WarningPrintf("track_data is not an array of object");
+        continue;
+      }
+
+      auto tracks = track_revisions.FindMember("tracks");
+      if (tracks == track_revisions.MemberEnd())
+      {
+        Log_WarningPrintf("tracks member is missing");
+        continue;
+      }
+
+      if (!tracks->value.IsArray())
+      {
+        Log_WarningPrintf("tracks is not an array");
+        continue;
+      }
+
+      std::string revisionString;
+      GetStringFromObject(track_revisions, "version", &revisionString);
+
+      for (const rapidjson::Value& track : tracks->value.GetArray())
+      {
+        auto md5_field = track.FindMember("md5");
+        if (md5_field == track.MemberEnd() || !md5_field->value.IsString())
+        {
+          continue;
+        }
+
+        auto md5 = CDImageHasher::HashFromString(
+          std::string_view(md5_field->value.GetString(), md5_field->value.GetStringLength()));
+        if (md5)
+        {
+          result.emplace(std::piecewise_construct, std::forward_as_tuple(md5.value()),
+                         std::forward_as_tuple(codes, revisionString, revision));
+        }
+      }
+      revision++;
+    }
+  }
+  return result;
+}
+
+bool GameDatabase::GetEntryForDisc(CDImage* image, GameDatabaseEntry* entry) const
 {
   std::string exe_name_code(System::GetGameCodeForImage(image, false));
   if (!exe_name_code.empty() && GetEntryForCode(exe_name_code, entry))

src/frontend-common/game_database.h

@@ -1,10 +1,9 @@
 #pragma once
+#include "common/cd_image_hasher.h"
 #include "core/types.h"
-#include <memory>
-#include <optional>
+#include <map>
 #include <string>
 #include <string_view>
-#include <unordered_map>
 #include <vector>
 
 class CDImage;
@@ -33,12 +32,31 @@ public:
   bool Load();
   void Unload();
 
-  bool GetEntryForDisc(CDImage* image, GameDatabaseEntry* entry);
-
-  bool GetEntryForCode(const std::string_view& code, GameDatabaseEntry* entry);
-
-  bool GetTitleAndSerialForDisc(CDImage* image, GameDatabaseEntry* entry);
-  //bool Get
+  bool GetEntryForDisc(CDImage* image, GameDatabaseEntry* entry) const;
+
+  bool GetEntryForCode(const std::string_view& code, GameDatabaseEntry* entry) const;
+
+  // Map of track hashes for image verification
+  struct TrackData
+  {
+    TrackData(std::vector<std::string> codes, std::string revisionString, uint32_t revision)
+      : codes(codes), revisionString(revisionString), revision(revision)
+    {
+    }
+
+    friend bool operator==(const TrackData& left, const TrackData& right)
+    {
+      // 'revisionString' is deliberately ignored in comparisons as it's redundant with comparing 'revision'! Do not
+      // change!
+      return left.codes == right.codes && left.revision == right.revision;
+    }
+
+    std::vector<std::string> codes;
+    std::string revisionString;
+    uint32_t revision;
+  };
+  using TrackHashesMap = std::multimap<CDImageHasher::Hash, TrackData>;
+  TrackHashesMap GetTrackHashesMap() const;
 
 private:
   void* m_json = nullptr;