diff --git a/src/robots.cpp b/src/robots.cpp index 1c6c548..9e43524 100644 --- a/src/robots.cpp +++ b/src/robots.cpp @@ -55,7 +55,7 @@ namespace Rep Robots::Robots(const std::string& content): agents_(), sitemaps_(), default_(agents_["*"]) { - std::string agent_name(""); + std::string agent_name("*"); std::istringstream input(content); if (content.compare(0, 3, "\xEF\xBB\xBF") == 0) { @@ -103,30 +103,14 @@ namespace Rep } else if (key.compare("disallow") == 0) { - if (agent_name.empty()) - { - throw std::invalid_argument( - "Need User-Agent before Disallow"); - } current->second.disallow(value); } else if (key.compare("allow") == 0) { - if (agent_name.empty()) - { - throw std::invalid_argument( - "Need User-Agent before Allow"); - } current->second.allow(value); } else if (key.compare("crawl-delay") == 0) { - if (agent_name.empty()) - { - throw std::invalid_argument( - "Need User-Agent before Crawl-Delay"); - } - try { current->second.delay(std::stof(value)); diff --git a/test/test-robots.cpp b/test/test-robots.cpp index ce9f58a..a356e76 100644 --- a/test/test-robots.cpp +++ b/test/test-robots.cpp @@ -2,22 +2,17 @@ #include "robots.h" -TEST(RobotsTest, DisallowFirst) +TEST(RobotsTest, NoLeadingUserAgent) { - // Disallow must be preceeded by a User-Agent line - ASSERT_THROW(Rep::Robots("Disallow: /path"), std::invalid_argument); -} - -TEST(RobotsTest, AllowFirst) -{ - // Allow must be preceeded by a User-Agent line - ASSERT_THROW(Rep::Robots("Allow: /path"), std::invalid_argument); -} - -TEST(RobotsTest, CrawlDelayFirst) -{ - // Crawl-delay must be preceeded by a User-Agent line - ASSERT_THROW(Rep::Robots("Crawl-delay: 1"), std::invalid_argument); + // Assumed to be the default user agent + std::string content = + "Disallow: /path\n" + "Allow: /path/exception\n" + "Crawl-delay: 5.2\n"; + Rep::Robots robot(content); + EXPECT_TRUE(robot.allowed("/path/exception", "agent")); + EXPECT_FALSE(robot.allowed("/path", "agent")); + EXPECT_NEAR(robot.agent("agent").delay(), 5.2, 0.000001); } TEST(RobotsTest, WellFormedCrawlDelay)