Menu
▾
▴
[Flightgear-commitlogs] [SimGear] branch release/2024.1 updated: Use dirindex.txt for TerraSync
|
From: <fli...@li...> - 2026-07-29 07:21:09
|
unknown user pushed a commit to branch release/2024.1
in repository simgear.
The following commit(s) were added to refs/heads/release/2024.1 by this push:
new 8f6d8817 Use dirindex.txt for TerraSync
8f6d8817 is described below
SF URL: http://sourceforge.net/p/flightgear/simgear/ci/8f6d88172ab8d8f6497b24032db096a7dfda3c6d/
Commit: 8f6d88172ab8d8f6497b24032db096a7dfda3c6d
Author: James Turner
Committer: James Turner
AuthorDate: Mon Jul 27 16:24:49 2026 +0100
Use dirindex.txt for TerraSync
---
simgear/io/HTTPRepository.cxx | 121 ++++++++++++++-------
simgear/io/test_repository.cxx | 239 ++++++++++++++++++++++++++++++++++++++++-
2 files changed, 321 insertions(+), 39 deletions(-)
diff --git a/simgear/io/HTTPRepository.cxx b/simgear/io/HTTPRepository.cxx
index 3024ac73..2c66c835 100644
--- a/simgear/io/HTTPRepository.cxx
+++ b/simgear/io/HTTPRepository.cxx
@@ -62,6 +62,10 @@ using namespace std::string_literals;
namespace {
+const auto DIR_INDEX_FILENAME = "dirindex.txt"s;
+const auto LEGACY_DIR_INDEX_FILENAME = ".dirindex"s;
+const auto DIR_HASH_FILENAME = ".dirhash"s;
+
std::string innerResultCodeAsString(HTTPRepository::ResultCode code) {
switch (code) {
case HTTPRepository::REPO_NO_ERROR:
@@ -229,9 +233,7 @@ public:
didCheck();
- SGPath fpath(absolutePath());
- fpath.append(".dirindex");
- updatedFileContents(fpath, "SELF", hash);
+ updatedFileContents(indexFilePath(true), "SELF", hash);
children.clear();
parseDirIndex(children);
@@ -241,6 +243,22 @@ public:
_repository->lastUpdatedDir = _relativePath;
}
+ SGPath indexFilePath(bool forceNewName) const
+ {
+ SGPath p = absolutePath() / DIR_INDEX_FILENAME;
+ if (forceNewName) {
+ return p;
+ }
+
+ if (p.exists()) {
+ return p;
+ }
+
+ // legacy file name
+ p = absolutePath() / LEGACY_DIR_INDEX_FILENAME;
+ return p;
+ }
+
void failedToUpdate(HTTPRepository::ResultCode status, const std::string& details)
{
if (_relativePath.empty()) {
@@ -418,7 +436,7 @@ public:
_repository->scheduleUpdateOfChildren(childDir);
}
}
- } // of repository-defined (well, .dirIndex) children iteration
+ } // of repository-defined (well, dirIndex) children iteration
// allow the filtering of orphans; this is important so that a filter
// can be used to preserve non-repo files in a directory,
@@ -471,8 +489,15 @@ public:
void removeOrphans(const PathList orphans)
{
for (const auto& o : orphans) {
- if (o.file() == ".dirindex"s) continue;
- if (o.file() == ".dirhash"s) continue;
+ // Never treat either index-file name as an orphan: during the
+ // migration period a directory may have both 'dirindex.txt' (newly
+ // written) and '.dirindex' (left over from an older client) present
+ // at the same time. Deleting the legacy file here would be safe,
+ // but is deliberately deferred so that the transition can be rolled
+ // back without data loss.
+ if (o.file() == DIR_INDEX_FILENAME) continue;
+ if (o.file() == LEGACY_DIR_INDEX_FILENAME) continue;
+ if (o.file() == DIR_HASH_FILENAME) continue;
removeChild(o);
}
}
@@ -761,7 +786,7 @@ private:
bool parseDirIndex(ChildInfoList& children)
{
- const SGPath p = absolutePath() / ".dirindex";
+ const auto p = indexFilePath(false);
if (!p.exists()) {
return false;
}
@@ -787,13 +812,13 @@ private:
if( typeData == "version" ) {
if( tokens.size() < 2 ) {
- SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: missing version number in line '" << line << "'"
- << "\n\tparsing:" << p.utf8Str());
+ SG_LOG(SG_TERRASYNC, SG_WARN, "malformed dirindex file: missing version number in line '" << line << "'"
+ << "\n\tparsing:" << p.utf8Str());
break;
}
if( tokens[1] != "1" ) {
- SG_LOG(SG_TERRASYNC, SG_WARN, "invalid .dirindex file: wrong version number '" << tokens[1] << "' (expected 1)"
- << "\n\tparsing:" << p.utf8Str());
+ SG_LOG(SG_TERRASYNC, SG_WARN, "invalid dirindex file: wrong version number '" << tokens[1] << "' (expected 1)"
+ << "\n\tparsing:" << p.utf8Str());
break;
}
continue; // version is good, continue
@@ -804,27 +829,27 @@ private:
}
if( typeData == "time" && tokens.size() > 1 ) {
- // SG_LOG(SG_TERRASYNC, SG_INFO, ".dirindex at '" << p.str() << "' timestamp: " << tokens[1] );
+ // SG_LOG(SG_TERRASYNC, SG_INFO, "dirindex at '" << p.str() << "' timestamp: " << tokens[1] );
continue;
}
if( tokens.size() < 3 ) {
- SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: not enough tokens in line '" << line << "' (ignoring line)"
- << "\n\tparsing:" << p.utf8Str());
+ SG_LOG(SG_TERRASYNC, SG_WARN, "malformed dirindex file: not enough tokens in line '" << line << "' (ignoring line)"
+ << "\n\tparsing:" << p.utf8Str());
continue;
}
if (typeData != "f" && typeData != "d" && typeData != "t" ) {
- SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: invalid type in line '" << line << "', expected 't', 'd' or 'f', (ignoring line)"
- << "\n\tparsing:" << p.utf8Str());
+ SG_LOG(SG_TERRASYNC, SG_WARN, "malformed dirindex file: invalid type in line '" << line << "', expected 't', 'd' or 'f', (ignoring line)"
+ << "\n\tparsing:" << p.utf8Str());
continue;
}
// security: prevent writing outside the repository via ../../.. filenames
- // (valid filenames never contain / - subdirectories have their own .dirindex)
+ // (valid filenames never contain / - subdirectories have their own dirindex)
if ((tokens[1] == "..") || (tokens[1].find_first_of("/\\") != std::string::npos)) {
- SG_LOG(SG_TERRASYNC, SG_WARN, "malformed .dirindex file: invalid filename in line '" << line << "', (ignoring line)"
- << "\n\tparsing:" << p.utf8Str());
+ SG_LOG(SG_TERRASYNC, SG_WARN, "malformed dirindex file: invalid filename in line '" << line << "', (ignoring line)"
+ << "\n\tparsing:" << p.utf8Str());
continue;
}
@@ -869,7 +894,10 @@ private:
{
SGPath p(child.path);
if (child.type == HTTPRepository::DirectoryType) {
- p.append(".dirindex");
+ p.append(DIR_INDEX_FILENAME);
+ if (!p.exists()) {
+ p = child.path / LEGACY_DIR_INDEX_FILENAME;
+ }
}
return hashForPath(p, child.name);
}
@@ -879,7 +907,7 @@ private:
* Previously we encoded the file absolute path, but this is annoying when files
* are relocated on disk. Now we store just the relative file name, and special value
* 'SELF' for the directory itself.
- *
+ *
* @param path candidate path from existing dirhash entry
* @return std::string updated hask entry key
*/
@@ -961,7 +989,7 @@ private:
// to the file in the future if needed
continue;
}
-
+
const auto nameData = fixupEntryName(simgear::strutils::strip(tokens[0]));
const std::string timeData = simgear::strutils::strip(tokens[1]);
const std::string sizeData = simgear::strutils::strip(tokens[2]);
@@ -1413,6 +1441,16 @@ void HTTPRepoGetRequest::cancel()
_directory->didCheck();
_directory->updateChildrenAfterRefresh();
} else if (responseCode() == 404) {
+ // temporary, remove once all servers / mirrors have the new names for the index files
+ if (!_didTryLegacyName) {
+ _didTryLegacyName = true;
+ // go around again, with the fallback URL
+ setUrl(_directory->url() + "/" + LEGACY_DIR_INDEX_FILENAME);
+ SG_LOG(SG_TERRASYNC, SG_INFO, "TerraSync: falling back to legacy dir-index name for:" << url());
+ _directory->repository()->finishedRequest(this, HTTPRepoPrivate::RequestFinish::Retry);
+ return;
+ }
+
_directory->failedToUpdate(
HTTPRepository::REPO_ERROR_FILE_NOT_FOUND, "Server returned 404/NOT FOUND");
} else if (responseCode() == 304) {
@@ -1427,36 +1465,43 @@ void HTTPRepoGetRequest::cancel()
this, HTTPRepoPrivate::RequestFinish::Done);
}
- void onFail() override {
- SG_LOG(SG_TERRASYNC, SG_ALERT, "onFail(): url()=" << url() << " _directory=" << _directory
- << " responseCode()=" << responseCode());
- HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
- if (responseCode() == -1) {
- code = HTTPRepository::REPO_ERROR_CANCELLED;
- }
+ void onFail() override
+ {
+ const auto rc = responseCode();
+ SG_LOG(SG_TERRASYNC, SG_ALERT, "onFail(): url()=" << url() << " _directory=" << _directory << " responseCode()=" << rc);
+ HTTPRepository::ResultCode code = HTTPRepository::REPO_ERROR_SOCKET;
+ if (rc == -1) {
+ code = HTTPRepository::REPO_ERROR_CANCELLED;
+ }
- if (_directory) {
- _directory->failedToUpdate(code, "HTTP layer failed request for:"s + url());
- const auto doRetry = HTTPRepoPrivate::RequestFinish::Done;
- _directory->repository()->finishedRequest(this, doRetry);
- }
+ if (_directory) {
+ _directory->failedToUpdate(code, "HTTP layer failed request for:"s + url());
+ const auto doRetry = HTTPRepoPrivate::RequestFinish::Done;
+ _directory->repository()->finishedRequest(this, doRetry);
+ }
}
private:
static std::string makeUrl(HTTPDirectory* d)
{
- return d->url() + "/.dirindex";
+ return d->url() + "/" + DIR_INDEX_FILENAME;
}
SGPath pathInRepo() const
{
- SGPath p(_directory->absolutePath());
- p.append(".dirindex");
- return p;
+ // always use the new name, not the legacy name here, so we write
+ // downloaded data to the correct location.
+ return _directory->indexFilePath(true);
}
simgear::sha1nfo hashContext;
std::string body;
+ // Tracks whether we have already retried this request with the legacy
+ // '.dirindex' URL after a 404 on 'dirindex.txt'. Not reset by
+ // prepareForRetry() intentionally: on a socket-failure retry the URL
+ // is preserved as-is, so a subsequent 404 on the legacy URL correctly
+ // reports NOT_FOUND rather than triggering another fallback attempt.
+ bool _didTryLegacyName = false;
bool _isRootDir; ///< is this the repository root?
std::string _targetHash;
};
diff --git a/simgear/io/test_repository.cxx b/simgear/io/test_repository.cxx
index c1ad9a34..c33db42a 100644
--- a/simgear/io/test_repository.cxx
+++ b/simgear/io/test_repository.cxx
@@ -59,6 +59,10 @@ std::string hashForData(const std::string& d)
class TestRepoEntry;
using AccessCallback = std::function<void(TestRepoEntry &entry)>;
+enum class DirIndexServerMode { NewStyle,
+ LegacyStyle };
+DirIndexServerMode g_dirIndexMode = DirIndexServerMode::NewStyle;
+
class TestRepoEntry
{
public:
@@ -261,7 +265,7 @@ public:
std::string repoPath = path.substr(6);
bool lookingForDir = false;
- std::string::size_type suffix = repoPath.find(".dirindex");
+ std::string::size_type suffix = repoPath.find(g_dirIndexMode == DirIndexServerMode::LegacyStyle ? ".dirindex" : "dirindex.txt");
if (suffix != std::string::npos) {
lookingForDir = true;
if (suffix > 0) {
@@ -380,6 +384,62 @@ void verifyFileNotPresent(const SGPath& fsRoot, const std::string& relPath)
}
}
+// Check that a directory within a local repo clone has the expected index file
+// name. Pass expectNewName=true to assert 'dirindex.txt' is present, or false
+// to assert the legacy '.dirindex' is present.
+void verifyIndexFileName(const SGPath& fsRoot, const std::string& relDir, bool expectNewName)
+{
+ SGPath dirPath(fsRoot);
+ if (!relDir.empty()) {
+ dirPath.append(relDir);
+ }
+
+ const SGPath newNamePath = dirPath / "dirindex.txt";
+ const SGPath oldNamePath = dirPath / ".dirindex";
+ const std::string label = relDir.empty() ? "(root)" : relDir;
+
+ if (expectNewName) {
+ if (!newNamePath.exists()) {
+ throw sg_error("Expected dirindex.txt but not found in dir", label);
+ }
+ } else {
+ if (!oldNamePath.exists()) {
+ throw sg_error("Expected .dirindex but not found in dir", label);
+ }
+ // Also assert the new name has NOT been written, so that tests which
+ // check "this directory was not re-fetched" are not vacuously satisfied
+ // by a bug that writes dirindex.txt everywhere.
+ if (newNamePath.exists()) {
+ throw sg_error("Found unexpected dirindex.txt in dir expected to keep legacy name", label);
+ }
+ }
+}
+
+// Recursively rename every 'dirindex.txt' to '.dirindex' and delete every
+// '.dirhash' under 'path'. Used to simulate a local clone that was last
+// written by an older client that stored index files with the legacy name.
+void convertToLegacyIndexFiles(const SGPath& path)
+{
+ SGPath newIndex = path / "dirindex.txt";
+ SGPath oldIndex = path / ".dirindex";
+ SGPath hashFile = path / ".dirhash";
+
+ if (newIndex.exists()) {
+ if (!newIndex.rename(oldIndex)) {
+ throw sg_io_exception("convertToLegacyIndexFiles: failed to rename dirindex.txt", path);
+ }
+ }
+ if (hashFile.exists()) {
+ hashFile.remove();
+ }
+
+ simgear::Dir d(path);
+ PathList subdirs = d.children(Dir::TYPE_DIR | Dir::NO_DOT_OR_DOTDOT);
+ for (const auto& subdir : subdirs) {
+ convertToLegacyIndexFiles(subdir);
+ }
+}
+
void verifyRequestCount(const std::string& relPath, int count)
{
TestRepoEntry* entry = global_repo->findEntry(relPath);
@@ -832,6 +892,7 @@ void testRetryAfterSocketFailure(HTTP::Client *cl) {
int aaFailsRemaining = 2;
int subdirBAFailsRemaining = 2;
+
TestApi::setResponseDoneCallback(
cl, [&aaFailsRemaining, &subdirBAFailsRemaining](int curlResult,
HTTP::Request_ptr req) {
@@ -871,6 +932,176 @@ void testRetryAfterSocketFailure(HTTP::Client *cl) {
verifyRequestCount("dirB/subdirA/fileBAC", 1);
}
+// Scenario 1: old server (serves .dirindex), new client code, fresh clone.
+// The client falls back from the new name to the legacy name on the wire, but
+// always persists the downloaded index under the new 'dirindex.txt' name.
+void testLegacyServerFreshClone(HTTP::Client* cl)
+{
+ global_repo->clearRequestCounts();
+ global_repo->clearFailFlags();
+ g_dirIndexMode = DirIndexServerMode::LegacyStyle;
+
+ SGPath p(simgear::Dir::current().path());
+ p.append("http_repo_legacy_server_fresh");
+ simgear::Dir pd(p);
+ if (pd.exists()) {
+ pd.removeChildren();
+ }
+
+ std::unique_ptr<HTTPRepository> repo(new HTTPRepository(p, cl));
+ repo->setBaseUrl("http://localhost:2000/repo");
+ repo->update();
+ waitForUpdateComplete(cl, repo.get());
+
+ g_dirIndexMode = DirIndexServerMode::NewStyle;
+
+ if (repo->failure() != HTTPRepository::REPO_NO_ERROR) {
+ throw sg_exception("Legacy server fresh clone failed with error: " +
+ HTTPRepository::resultCodeAsString(repo->failure()));
+ }
+
+ verifyFileState(p, "fileA");
+ verifyFileState(p, "dirB/subdirA/fileBAA");
+ verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
+
+ // Even though the server only served .dirindex, the client must save the
+ // index under the new name at all directory levels.
+ verifyIndexFileName(p, "", true);
+ verifyIndexFileName(p, "dirA", true);
+ verifyIndexFileName(p, "dirB", true);
+ verifyIndexFileName(p, "dirB/subdirA", true);
+ verifyIndexFileName(p, "dirC", true);
+
+ std::cout << "Passed test: legacy server fresh clone uses new local index file names" << std::endl;
+}
+
+// Scenario 2: existing local clone that has old-style '.dirindex' files.
+// After an update in which some directories change, those directories have
+// their index file migrated to the new name; unchanged directories keep the
+// legacy name.
+void testLegacyLocalFilesUpdate(HTTP::Client* cl)
+{
+ global_repo->clearRequestCounts();
+ global_repo->clearFailFlags();
+
+ SGPath p(simgear::Dir::current().path());
+ p.append("http_repo_legacy_local_update");
+ simgear::Dir pd(p);
+ if (pd.exists()) {
+ pd.removeChildren();
+ }
+
+ // Step 1: fresh clone with new-style server to populate the local tree.
+ {
+ std::unique_ptr<HTTPRepository> repo(new HTTPRepository(p, cl));
+ repo->setBaseUrl("http://localhost:2000/repo");
+ repo->update();
+ waitForUpdateComplete(cl, repo.get());
+
+ if (repo->failure() != HTTPRepository::REPO_NO_ERROR) {
+ throw sg_exception("Initial clone in legacy local files test failed");
+ }
+ }
+ cl->clearAllConnections();
+
+ // Step 2: simulate a clone that was last written by an old client by
+ // renaming all index files to the legacy name and clearing hash caches.
+ // Without clearing the caches the stale entries for 'dirindex.txt' would
+ // make every directory look out-of-date regardless of actual content.
+ convertToLegacyIndexFiles(p);
+
+ verifyIndexFileName(p, "", false);
+ verifyIndexFileName(p, "dirA", false);
+ verifyIndexFileName(p, "dirB/subdirA", false);
+
+ // Step 3: modify a subset of the server-side repo.
+ global_repo->findEntry("dirB/subdirA/fileBAA")->revision++;
+ global_repo->defineFile("dirC/fileCX");
+ global_repo->clearRequestCounts();
+
+ // Step 4: update using the legacy server. The client first tries
+ // 'dirindex.txt' (404), then falls back to '.dirindex' (200) for every
+ // directory it actually fetches. The downloaded content is always saved
+ // under the new name.;
+ g_dirIndexMode = DirIndexServerMode::LegacyStyle;
+
+ {
+ std::unique_ptr<HTTPRepository> repo(new HTTPRepository(p, cl));
+ repo->setBaseUrl("http://localhost:2000/repo");
+ repo->update();
+ waitForUpdateComplete(cl, repo.get());
+
+ // Reset before any assertions that can throw.
+ g_dirIndexMode = DirIndexServerMode::NewStyle;
+
+ if (repo->failure() != HTTPRepository::REPO_NO_ERROR) {
+ throw sg_exception("Update in legacy local files test failed");
+ }
+ }
+
+ // Clean up the repo entry added for this test before the assertions so it
+ // also runs on failure paths where assertions throw. Note: verifyFileState
+ // reads from global_repo, so fileCX must still be present when we call it.
+ // We verify it first, then clean up.
+ verifyFileState(p, "dirB/subdirA/fileBAA");
+ verifyFileState(p, "dirC/fileCX");
+
+ // Directories whose content changed were re-fetched and must now have the
+ // new index file name.
+ verifyIndexFileName(p, "", true); // root: dirB and dirC hashes changed
+ verifyIndexFileName(p, "dirB", true); // subdirA hash changed
+ verifyIndexFileName(p, "dirB/subdirA", true); // fileBAA was bumped
+ verifyIndexFileName(p, "dirC", true); // fileCX was added
+
+ // Directories that were not touched keep the legacy name (and must NOT
+ // have had dirindex.txt written — verifyIndexFileName checks both sides
+ // when expectNewName=false).
+ verifyIndexFileName(p, "dirA", false);
+ verifyIndexFileName(p, "dirB/subdirB", false);
+
+ // Clean up the repo entry added for this test to avoid polluting later tests.
+ global_repo->findEntry("dirC")->removeChild("fileCX");
+
+ std::cout << "Passed test: update with legacy local index files migrates changed dirs to new name" << std::endl;
+}
+
+// Scenario 3: new server (serves dirindex.txt), fresh clone. The happy path:
+// index files must be stored under the new name at every level.
+void testNewServerFreshClone(HTTP::Client* cl)
+{
+ global_repo->clearRequestCounts();
+ global_repo->clearFailFlags();
+
+ SGPath p(simgear::Dir::current().path());
+ p.append("http_repo_new_server_fresh");
+ simgear::Dir pd(p);
+ if (pd.exists()) {
+ pd.removeChildren();
+ }
+
+ // g_serverMode is already NewStyle
+ std::unique_ptr<HTTPRepository> repo(new HTTPRepository(p, cl));
+ repo->setBaseUrl("http://localhost:2000/repo");
+ repo->update();
+ waitForUpdateComplete(cl, repo.get());
+
+ if (repo->failure() != HTTPRepository::REPO_NO_ERROR) {
+ throw sg_exception("New server fresh clone failed");
+ }
+
+ verifyFileState(p, "fileA");
+ verifyFileState(p, "dirB/subdirA/fileBAA");
+ verifyFileState(p, "dirC/subdirA/subsubA/fileCAAA");
+
+ verifyIndexFileName(p, "", true);
+ verifyIndexFileName(p, "dirA", true);
+ verifyIndexFileName(p, "dirB", true);
+ verifyIndexFileName(p, "dirB/subdirA", true);
+ verifyIndexFileName(p, "dirC", true);
+
+ std::cout << "Passed test: new server fresh clone uses new index file names" << std::endl;
+}
+
void testPersistentSocketFailure(HTTP::Client *cl) {
global_repo->clearRequestCounts();
global_repo->clearFailFlags();
@@ -952,6 +1183,12 @@ int main(int argc, char* argv[])
testServer.disconnectAll();
cl.clearAllConnections();
+ testLegacyServerFreshClone(&cl);
+ testLegacyLocalFilesUpdate(&cl);
+ testNewServerFreshClone(&cl);
+
+ cl.clearAllConnections();
+
testServerModifyDuringSync(&cl);
testDestroyDuringSync(&cl);
|