Broad performance changes (draft - early comments welcome)

src/Context.cpp

@@ -1126,6 +1126,10 @@ void Context::staticInitialization() {
   for (auto& var : config.all())
     if (var.substr(0, 13) == "urgency.user." || var.substr(0, 12) == "urgency.uda.")
       Task::coefficients[var] = config.getReal(var);
+
+  // Pre-parse the coefficient keys, so urgency_c() doesn't have to re-parse for
+  // each task.
+  Task::setUrgencyCoefficients();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

src/Filter.cpp

@@ -44,31 +44,7 @@ void Filter::subset(const std::vector<Task>& input, std::vector<Task>& output) {
 
   Context::getContext().cli2.prepareFilter();
 
-  std::vector<std::pair<std::string, Lexer::Type>> precompiled;
-  for (auto& a : Context::getContext().cli2._args)
-    if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
-
-  if (precompiled.size()) {
-    Eval eval;
-    eval.addSource(domSource);
-
-    // Debug output from Eval during compilation is useful.  During evaluation
-    // it is mostly noise.
-    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3 ? true : false);
-    eval.compileExpression(precompiled);
-
-    for (auto& task : input) {
-      // Set up context for any DOM references.
-      auto currentTask = Context::getContext().withCurrentTask(&task);
-
-      Variant var;
-      eval.evaluateCompiledExpression(var);
-      if (var.get_bool()) output.push_back(task);
-    }
-
-    eval.debug(false);
-  } else
-    output = input;
+  filter_to_tasks(input, output);
 
   _endCount = (int)output.size();
   Context::getContext().debug(
@@ -86,51 +62,28 @@ void Filter::subset(std::vector<Task>& output) {
   for (auto& a : Context::getContext().cli2._args)
     if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
 
-  // Shortcut indicates that only pending.data needs to be loaded.
+  // Shortcut indicates that only tasks in the working set are loaded.
   bool shortcut = false;
 
   if (precompiled.size()) {
     Timer timer_pending;
-    auto pending = Context::getContext().tdb2.pending_tasks();
+    const auto& pending = Context::getContext().tdb2.pending_tasks();
     Context::getContext().time_filter_us -= timer_pending.total_us();
     _startCount = (int)pending.size();
 
-    Eval eval;
-    eval.addSource(domSource);
-
-    // Debug output from Eval during compilation is useful.  During evaluation
-    // it is mostly noise.
-    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3 ? true : false);
-    eval.compileExpression(precompiled);
-
     output.clear();
-    for (auto& task : pending) {
-      // Set up context for any DOM references.
-      auto currentTask = Context::getContext().withCurrentTask(&task);
 
-      Variant var;
-      eval.evaluateCompiledExpression(var);
-      if (var.get_bool()) output.push_back(task);
-    }
+    filter_to_tasks(pending, output);
 
     shortcut = pendingOnly();
     if (!shortcut) {
       Timer timer_completed;
-      auto completed = Context::getContext().tdb2.completed_tasks();
+      const auto& completed = Context::getContext().tdb2.completed_tasks();
       Context::getContext().time_filter_us -= timer_completed.total_us();
       _startCount += (int)completed.size();
 
-      for (auto& task : completed) {
-        // Set up context for any DOM references.
-        auto currentTask = Context::getContext().withCurrentTask(&task);
-
-        Variant var;
-        eval.evaluateCompiledExpression(var);
-        if (var.get_bool()) output.push_back(task);
-      }
+      filter_to_tasks(completed, output);
     }
-
-    eval.debug(false);
   } else {
     safety();
 
@@ -145,73 +98,110 @@ void Filter::subset(std::vector<Task>& output) {
   Context::getContext().time_filter_us += timer.total_us();
 }
 
-////////////////////////////////////////////////////////////////////////////////
+/////////////////////////////////////////////////////////////////////////////////
 bool Filter::hasFilter() const {
   for (const auto& a : Context::getContext().cli2._args)
     if (a.hasTag("FILTER")) return true;
 
   return false;
 }
 
+/////////////////////////////////////////////////////////////////////////////////
+// Evaluates a pre-parsed filter against a set of tasks and stores their indices
+// from the vector. The filter is parsed with prepareFilter(), but this
+// function does not call that or safety() itself - callers are expected to do so.
+void Filter::filter_to_indices(const std::vector<Task>& pending, std::vector<int>& indices) const {
+  std::vector<std::pair<std::string, Lexer::Type>> precompiled;
+  for (auto& a : Context::getContext().cli2._args)
+    if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
+
+  if (precompiled.empty()) {
+    indices.reserve(pending.size());
+    for (int i = 0; i < (int)pending.size(); ++i) indices.push_back(i);
+  } else {
+    Eval eval;
+    eval.addSource(domSource);
+    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3);
+    eval.compileExpression(precompiled);
+    for (int i = 0; i < (int)pending.size(); ++i) {
+      auto currentTask = Context::getContext().withCurrentTask(&pending[i]);
+      Variant var;
+      eval.evaluateCompiledExpression(var);
+      if (var.get_bool()) indices.push_back(i);
+    }
+    eval.debug(false);
+  }
+
+  Context::getContext().debug(
+      format("Filtered {1} tasks --> {2} tasks [pending only]", pending.size(), indices.size()));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Like filter_to_indices, but copies matched tasks into the output.
+void Filter::filter_to_tasks(const std::vector<Task>& input, std::vector<Task>& output) const {
+  std::vector<std::pair<std::string, Lexer::Type>> precompiled;
+  for (auto& a : Context::getContext().cli2._args)
+    if (a.hasTag("FILTER")) precompiled.emplace_back(a.getToken(), a._lextype);
+
+  if (precompiled.empty()) {
+    output = input;
+  } else {
+    Eval eval;
+    eval.addSource(domSource);
+    eval.debug(Context::getContext().config.getInteger("debug.parser") >= 3);
+    eval.compileExpression(precompiled);
+    for (auto& task : input) {
+      auto currentTask = Context::getContext().withCurrentTask(&task);
+      Variant var;
+      eval.evaluateCompiledExpression(var);
+      if (var.get_bool()) output.push_back(task);
+    }
+    eval.debug(false);
+  }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 // If the filter contains no 'or', 'xor' or 'not' operators, and only includes
 // status values 'pending', 'waiting' or 'recurring', then the filter is
 // guaranteed to only need data from pending.data.
+
 bool Filter::pendingOnly() const {
-  // When GC is off, there are no shortcuts.
   if (!Context::getContext().config.getBoolean("gc")) return false;
 
-  // To skip loading completed.data, there should be:
-  // - 'status' in filter
-  // - no 'completed'
-  // - no 'deleted'
-  // - no 'xor'
-  // - no 'or'
-  int countStatus = 0;
-  int countPending = 0;
-  int countWaiting = 0;
-  int countRecurring = 0;
-  int countId = (int)Context::getContext().cli2._id_ranges.size();
-  int countUUID = (int)Context::getContext().cli2._uuid_list.size();
-  int countOr = 0;
-  int countXor = 0;
-  int countNot = 0;
-  bool pendingTag = false;
-  bool activeTag = false;
-
-  for (const auto& a : Context::getContext().cli2._args) {
-    if (a.hasTag("FILTER")) {
-      std::string raw = a.attribute("raw");
-      std::string canonical = a.attribute("canonical");
-
-      if (a._lextype == Lexer::Type::op && raw == "or") ++countOr;
-      if (a._lextype == Lexer::Type::op && raw == "xor") ++countXor;
-      if (a._lextype == Lexer::Type::op && raw == "not") ++countNot;
-      if (a._lextype == Lexer::Type::dom && canonical == "status") ++countStatus;
-      if (raw == "pending") ++countPending;
-      if (raw == "waiting") ++countWaiting;
-      if (raw == "recurring") ++countRecurring;
-    }
-  }
+  const auto& cli = Context::getContext().cli2;
+  if (!cli._uuid_list.empty()) return false;
 
-  for (const auto& word : Context::getContext().cli2._original_args) {
-    if (word.attribute("raw") == "+PENDING") pendingTag = true;
-    if (word.attribute("raw") == "+ACTIVE") activeTag = true;
-  }
-
-  if (countUUID) return false;
+  std::vector<const A2*> filter_args;
+  for (const auto& arg : cli._args) {
+    if (!arg.hasTag("FILTER")) continue;
 
-  if (countOr || countXor || countNot) return false;
+    const auto& raw = arg.attribute("raw");
+    if (arg._lextype == Lexer::Type::op &&
+        (raw == "or" || raw == "xor" || raw == "!" || raw == "not"))
+      return false;
 
-  if (pendingTag || activeTag) return true;
-
-  if (countStatus) {
-    if (!countPending && !countWaiting && !countRecurring) return false;
+    filter_args.push_back(&arg);
+  }
 
-    return true;
+  for (size_t i = 0; i + 2 < filter_args.size(); ++i) {
+    const auto& left = *filter_args[i];
+    const auto& op = *filter_args[i + 1];
+    const auto& right = *filter_args[i + 2];
+    const auto& value = right.attribute("raw");
+
+    if (left._lextype == Lexer::Type::dom && left.attribute("canonical") == "status" &&
+        op._lextype == Lexer::Type::op &&
+        (op.attribute("raw") == "=" || op.attribute("raw") == "==") &&
+        (value == "pending" || value == "waiting" || value == "recurring"))
+      return true;
+
+    if (left._lextype == Lexer::Type::dom && left.attribute("raw") == "tags" &&
+        op._lextype == Lexer::Type::op && op.attribute("raw") == "_hastag_" &&
+        (value == "PENDING" || value == "ACTIVE" || value == "READY" || value == "WAITING"))
+      return true;
   }
 
-  if (countId) return true;
+  if (!cli._id_ranges.empty()) return true;
 
   return false;
 }

src/Filter.h

@@ -40,6 +40,13 @@ class Filter {
   void subset(std::vector<Task>&);
   bool hasFilter() const;
   bool pendingOnly() const;
+
+  // These evaluate already-prepared filters (ie. caller calls
+  // cli2.prepareFilter() and safety()). filter_to_indices() stores matching
+  // indices, filter_to_tasks() copies task objects.
+  void filter_to_indices(const std::vector<Task>&, std::vector<int>&) const;
+  void filter_to_tasks(const std::vector<Task>&, std::vector<Task>&) const;
+
   void safety() const;
   void disableSafety();
 

src/Hooks.cpp

@@ -279,6 +279,9 @@ void Hooks::onModify(Task& before, Task& after) const {
   if (!_enabled) return;
 
   Timer timer;
+  const auto id = after.id;
+  const auto is_blocked = after.is_blocked;
+  const auto is_blocking = after.is_blocking;
 
   std::vector<std::string> matchingScripts = scripts("on-modify");
   if (matchingScripts.size()) {
@@ -314,6 +317,9 @@ void Hooks::onModify(Task& before, Task& after) const {
     }
 
     after = Task(input[1]);
+    after.id = id;
+    after.is_blocked = is_blocked;
+    after.is_blocking = is_blocking;
   }
 
   Context::getContext().time_hooks_us += timer.total_us();
@@ -322,6 +328,9 @@ void Hooks::onModify(Task& before, Task& after) const {
 ////////////////////////////////////////////////////////////////////////////////
 std::vector<std::string> Hooks::list() const { return _scripts; }
 
+////////////////////////////////////////////////////////////////////////////////
+bool Hooks::hasOnModify() const { return _enabled && !scripts("on-modify").empty(); }
+
 ////////////////////////////////////////////////////////////////////////////////
 std::vector<std::string> Hooks::scripts(const std::string& event) const {
   std::vector<std::string> matching;

src/Hooks.h

@@ -41,6 +41,7 @@ class Hooks {
   void onExit() const;
   void onAdd(Task&) const;
   void onModify(Task&, Task&) const;
+  bool hasOnModify() const;
   std::vector<std::string> list() const;
 
  private: