Implemented the GenServer crawler. It's stable

logsrv/apps/logsrv_api/lib/logsrv_api.ex

@@ -2,41 +2,67 @@ defmodule LogsrvApi do
   @moduledoc """
   Documentation for `LogsrvApi`.
   """
-  alias LogsrvApi.{Filesystem, Journal, Page}
+  alias LogsrvApi.Filesystem
 
   @repo Filesystem
 
-  def pages do
+  def pages, do: @repo.list! :pages
-    @repo.all(Page)
+  def journals, do: @repo.list! :journals
+
+  def to_title(fd) do
+    fd
+    |> String.replace(~r/_/," ")
+    |> String.replace(~r/\.md$/,"")
+    |> String.replace("%2F","/")
+    |> String.capitalize
   end
-  def page(title) do
+  def to_slug(fd) do
-    @repo.get!(Page, title)
+    fd
+    |> String.downcase
+    |> String.replace(~r/_/,"-")
+    |> String.replace(~r/\..*/,"")
+    |> String.replace("%2F","_")
+  end
+  def to_path(fd), do: fd |> @repo.locate
+  def to_date(slug) do
+    Timex.parse!(slug, "%Y-%m-%d", :strftime)
   end
 
-  def journals do
+  def date_modified(path) do
-    @repo.all(Journal)
+    %{ctime: ctime} = (path |> File.stat!)
-  end
+    ctime
-  def journal(str) do
+    |> Timex.to_datetime
-    {:ok, date} =  Filesystem.to_date(str)
-    @repo.get!(Journal, date)
   end
 
-  def get_by_id(id) do
+  def split(data) do
-    # Try to guess whether it's a page or a journal
+    try do
-    {_, date} = Filesystem.to_date(id)
+      [frontmatter, markdown] = String.split(data, ~r/\n-{3,}\n/, parts: 2)
-    case date do
+      {parse_yaml(frontmatter), Earmark.as_html!(markdown)}
-      :invalid_format ->
+    rescue MatchError ->
-        IO.puts(id)
+      {:nil, Earmark.as_html!(data)}
-        page(id)
+    end
-      _ ->
+  end
-        IO.puts(id)
+  def parse_yaml(yaml) do
-        journal(date)
+    yaml
+    |> :yamerl_constr.string
+    |> List.flatten
+  end
+  def extract({props, content}, post) do
+    new_title = get_prop(props, "title") || post.title
+    %{post |
+      title: new_title |> to_string,
+      tags: get_prop(props, "tags"),
+      content: content}
+  end
+  def get_prop(props, key) do
+    if is_nil(props) do
+      nil
+    else
+      case :proplists.get_value(String.to_char_list(key), props) do
+        :undefined -> nil
+        x -> x
+      end
     end
   end
 
-  def read(entry) do
-    @repo.read(entry)
-  end
-
-
 end

logsrv/apps/logsrv_api/lib/logsrv_api/application.ex

@@ -7,10 +7,7 @@ defmodule LogsrvApi.Application do
 
   @impl true
   def start(_type, _args) do
-    children = [
+    children = []
-      # Starts a worker by calling: LogsrvApi.Worker.start_link(arg)
-      # {LogsrvApi.Worker, arg}
-    ]
 
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options

logsrv/apps/logsrv_api/lib/logsrv_api/crawler.ex

@@ -1,7 +1,16 @@
 defmodule LogsrvApi.Crawler do
   alias LogsrvApi.{Filesystem,Page,Journal}
 
-  def crawl do
+  def crawl(module) do
-    Filesystem.
+    Filesystem.dir(module.atom)
+    |> File.ls!
+    |> Filesystem.subdir(module.atom)
+    |> Enum.map(&module.init/1)
+    |> Enum.sort(&sort/2)
   end
+
+  def sort(a, b) do
+    Timex.compare(a.date, b.date) > 0
+  end
+
 end

logsrv/apps/logsrv_api/lib/logsrv_api/filesystem.ex

@@ -1,5 +1,51 @@
 defmodule LogsrvApi.Filesystem do
   alias LogsrvApi.{Journal, Page}
+  use GenServer
+
+  def start_link(_opts) do
+    GenServer.start_link(__MODULE__, :ok, [name: __MODULE__])
+  end
+
+  def init(:ok) do
+    pages = LogsrvApi.Crawler.crawl(Page)
+    journals = LogsrvApi.Crawler.crawl(Journal)
+    posts = %{pages: pages, journals: journals}
+    {:ok, posts}
+  end
+
+  def get_by_slug(slug) do
+    GenServer.call(__MODULE__, {:get_by_slug, slug})
+  end
+
+  def get_by_slug!(slug) do
+    {:ok, post} = get_by_slug(slug)
+    post
+  end
+
+  def list() do
+    %{
+      journals: list!(:journals),
+      pages: list!(:pages)
+    }
+  end
+  def list(type) do
+    GenServer.call(__MODULE__, {:list, type})
+  end
+  def list!(type) do
+    {:ok, files} = list(type)
+    files
+  end
+
+  def handle_call({:get_by_slug, slug}, _from, posts) do
+    case [posts.journals, posts.pages] |> List.flatten |> Enum.find(fn(x) -> x.slug == slug end) do
+      nil -> {:reply, {:error,:not_found}, posts}
+      page -> {:reply, {:ok, page}, posts}
+    end
+  end
+
+  def handle_call({:list, type}, _from, posts) do
+    {:reply, {:ok, posts |> Map.get(type)}, posts}
+  end
 
   def to_date(str) when is_bitstring(str) do
     str
@@ -17,51 +63,46 @@ defmodule LogsrvApi.Filesystem do
   def dir(subdir) do
     "#{dir()}/#{subdir}/"
   end
-  def locate(fd, module) do
+
-    dir(module.atom) <> fd
+  def locate(fd) do
+    page = dir(:pages) <> fd
+    journal = dir(:journals) <> fd
+    cond do
+      page |> File.exists? ->
+        page
+      journal |> File.exists? ->
+        journal
+      true ->
+        # Occasional race condition I haven't figured out.
+        # Just repeating the function seems to work.
+        locate(fd)
+      end
   end
 
-  def all(Page) do
+  def is_dir(fd) do
-    dir(:pages)
+    path = fd |> locate
-    |> File.ls!
+    %{type: type} = path |> File.stat!
-    |> Enum.map(fn(fd) ->
+    type == :directory
-      case fd |> File.ls do
-        {:error, _} ->
-          IO.puts("Error on #{fd}")
-          Page.init(fd)
-        pages ->
-          IO.puts(pages)
-          pages |> Enum.map(fn(page) ->
-            "pages/#{page}" |> Page.init
-          end)
-      end
-    end)
-    |> Enum.sort(&sort/2)
   end
+
+  def subdir(list, type) do
+    list
+    |> Enum.map(fn(fd) ->
+      if is_dir(fd) do
+        subdir(dir(type) <>
+        fd
+        |> File.ls!
+        |> Enum.map(fn(dir) ->
+          fd <> "/" <> dir
+        end), type)
+      else
+        fd
+      end
+    end) |> List.flatten
+  end
+
   def sort(a, b) do
     Timex.compare(a.date, b.date) > 0
   end
 
-  def all(Journal) do
-    dir(:journals)
-    |> File.ls!
-    |> Enum.sort |> Enum.reverse
-    |> Enum.map(fn(fd) ->
-      Journal.init(fd)
-    end)
-  end
-
-  def get!(Journal, date) do
-    Enum.find(all(Journal), fn(entry) -> entry.date === date end)
-  end
-  def get!(Page, filename) do
-    Enum.find(all(Page), fn(entry) -> entry.filename === filename end)
-  end
-
-  def read(entry) do
-    entry.type
-    |> locate(entry.filename)
-    |> File.read!
-  end
-
 end

logsrv/apps/logsrv_api/lib/logsrv_api/journal.ex

@@ -1,16 +1,19 @@
 defmodule LogsrvApi.Journal do
   alias LogsrvApi.{Filesystem,Page,Journal}
+  import LogsrvApi
+
+  defstruct slug: "", title: "", date: "", content: "", filename: "", tags: [:fun]
+  def atom, do: :journals
 
   def init(fd) do
-    {:ok, date} = fd |> String.replace(~r/_/,"-") |> String.replace(~r/\.md$/,"") |> Filesystem.to_date
+    page = %Journal{
-    tags = [:fun]
-
-    %{
-      type: Journal,
-      date: date,
       filename: fd,
-      tags: tags
+      title: fd |> to_title,
+      slug: fd |> to_slug,
+      date: fd |> to_slug |> to_date
     }
-  end
 
+    fd |> to_path |> File.read! |> split |> extract(page)
+
+  end
 end

logsrv/apps/logsrv_api/lib/logsrv_api/page.ex

@@ -1,5 +1,6 @@
 defmodule LogsrvApi.Page do
   alias LogsrvApi.{Filesystem,Page,Journal}
+  import LogsrvApi
 
   defstruct slug: "", title: "", date: "", content: "", filename: "", tags: [:fun]
   def atom, do: :pages
@@ -15,59 +16,4 @@ defmodule LogsrvApi.Page do
     fd |> to_path |> File.read! |> split |> extract(page)
 
   end
-
-  def to_title(fd) do
-    fd
-    |> String.replace(~r/_/," ")
-    |> String.replace(~r/\.md$/,"")
-    |> String.replace("%2F","/")
-    |> String.capitalize
-  end
-  def to_slug(fd) do
-    fd
-    |> String.replace(~r/_/,"-")
-    |> String.replace(~r/\.md$/,"")
-    |> String.replace("%2F","_")
-  end
-  def to_path(fd) do
-    fd
-    |> Filesystem.locate(Page)
-  end
-
-  def date_modified(path) do
-    %{ctime: ctime} = (path |> File.stat!)
-    ctime
-    |> NaiveDateTime.from_erl!()
-    |> DateTime.from_naive!("Etc/UTC")
-  end
-
-  defp split(data) do
-    try do
-      [frontmatter, markdown] = String.split(data, ~r/\n-{3,}\n/, parts: 2)
-      {parse_yaml(frontmatter), Earmark.as_html!(markdown)}
-    rescue MatchError ->
-      {:nil, Earmark.as_html!(data)}
-    end
-  end
-  def parse_yaml(yaml) do
-    yaml
-    |> :yamerl_constr.string
-    |> List.flatten
-  end
-  defp extract({props, content}, post) do
-    %{post |
-      title: get_prop(props, "title") |>  to_string ||  post.title,
-      tags: get_prop(props, "tags"),
-      content: content}
-  end
-  defp get_prop(props, key) do
-    if is_nil(props) do
-      :nil
-    else
-      case :proplists.get_value(String.to_char_list(key), props) do
-        :undefined -> nil
-        x -> x
-      end
-    end
-  end
 end

logsrv/apps/logsrv_web/assets/css/app.css

@@ -1,6 +1,5 @@
 /* This file is for your main application CSS */
 @import "./phoenix.css";
-@import "./bear.css";
 
 /* Alerts and form errors used by phx.new */
 .alert {

logsrv/apps/logsrv_web/assets/css/phoenix.css

@@ -55,8 +55,8 @@ select {
 /* Headers */
 header {
   width: 100%;
-  background: #002B36;
+  background: #eee;
-  border-bottom: 1px solid #eaeaea;
+  border-bottom: 4px solid #222;
   margin-bottom: 2rem;
 }
 header section {

logsrv/apps/logsrv_web/lib/logsrv_web/application.ex

@@ -12,7 +12,8 @@ defmodule LogsrvWeb.Application do
       LogsrvWeb.Telemetry,
       # Start the Endpoint (http/https),
       {Phoenix.PubSub, [name: LogsrvWeb.PubSub, adapter: Phoenix.PubSub.PG2]},
-      LogsrvWeb.Endpoint
+      LogsrvWeb.Endpoint,
+      {LogsrvApi.Filesystem, []}
       # Start a worker by calling: LogsrvWeb.Worker.start_link(arg)
       # {LogsrvWeb.Worker, arg}
     ]

logsrv/apps/logsrv_web/lib/logsrv_web/controllers/post_controller.ex

@@ -1,20 +1,32 @@
 defmodule LogsrvWeb.PostController do
   use LogsrvWeb, :controller
   import LogsrvApi
+  import LogsrvApi.Filesystem
 
   def index(conn, _params) do
-    render(conn, "index.html", pages: pages(), journals: journals())
+    %{journals: journals, pages: pages} = list()
+    render conn, "index.html", pages: pages, journals: journals
   end
 
   def show(conn, %{"id" => id}) do
-    post = id |> get_by_id
+    {status, post} = id |> String.downcase |> get_by_slug
-    content = post |> read
+    case status do
-    render(conn, "post.html", post: post, pages: pages(), journals: journals(), content: content)
+      :ok ->
+        render(conn, "post.html", post: post)
+      _ ->
+        not_found(conn)
+      end
   end
 
   def home(conn, _params) do
-    homepage = get_by_id("contents.md")
+    show(conn, %{"id" => "contents"})
-    content = homepage |> read
-    render(conn, "post.html", post: homepage, pages: pages(), journals: journals(), content: content)
   end
+
+
+  def not_found(conn) do
+    conn
+    |> put_status(:not_found)
+    |> render(LogsrvWeb.ErrorView, "404.html")
+  end
+
 end

logsrv/apps/logsrv_web/lib/logsrv_web/templates/layout/root.html.heex

@@ -7,7 +7,6 @@
     <meta name="csrf-token" content={csrf_token_value()}>
     <%= live_title_tag assigns[:page_title] || "LogsrvWeb", suffix: " · Phoenix Framework" %>
     <link phx-track-static rel="stylesheet" href={Routes.static_path(@conn, "/assets/app.css")}/>
-    <link phx-track-static rel="stylesheet" href={Routes.static_path(@conn, "/assets/bear.css")}/>
     <script defer phx-track-static type="text/javascript" src={Routes.static_path(@conn, "/assets/app.js")}></script>
   </head>
   <body><div id="root"><div class="theme-inner"><div id="app-container">
@@ -17,8 +16,8 @@
         <div>
         <ul>
         <li>  <b>Directory:</b> <%= LogsrvApi.Filesystem.dir %> </li>
-        <li>  <b>Journals:</b> <%= @pages |> length %> </li>
+        <li>  <b>Journals:</b> <%= pages |> length %> </li>
-        <li>  <b>Pages:</b> <%= @journals |> length %> </li>        
+        <li>  <b>Pages:</b> <%= journals |> length %> </li>        
         </ul></div>
       <div> <a href="/pages"><h3>Pages</h3></a> </div><div> <a href="/journals"><h3>Journals</h3></a> </div>        
         </section>    

logsrv/apps/logsrv_web/lib/logsrv_web/templates/post/index.html.heex

@@ -13,7 +13,7 @@
       <%= for journal <- @journals do %>
       <li>
         <a href={post_path journal}>
-          <strong><%= journal.date %></strong> (<%= journal.filename %>)
+          <strong><%= journal.date |> Calendar.strftime("%b %d, %Y") %></strong> (<%= journal.filename %>)
         </a>
       </li>
       <% end %>

logsrv/apps/logsrv_web/lib/logsrv_web/templates/post/post.html.heex

@@ -1,8 +1,6 @@
-<section class="row">
+<p><a href="/">&laquo; Back to index</a></p>
-  <article class="column">
+<article>
-    <h2><%= @post.date |> Calendar.strftime("%A, %b %d, %Y") %></h2>
+  <h1><%= @post.title %></h1>
-  </article>  
+  <p style='float:right;'><%= Calendar.strftime(@post.date, "%b %d, %Y") %></p>
-</section>
+  <%= raw(@post.content) %>
-<section class="row">
+</article>
-    <%= raw @content |> AlchemistMarkdown.to_html |> cleanup %>
-</section>

logsrv/apps/logsrv_web/lib/logsrv_web/views/layout_view.ex

@@ -4,4 +4,15 @@ defmodule LogsrvWeb.LayoutView do
   # Phoenix LiveDashboard is available only in development by default,
   # so we instruct Elixir to not warn if the dashboard route is missing.
   @compile {:no_warn_undefined, {Routes, :live_dashboard_path, 2}}
+
+
+  def pages do
+    {:ok, pages} = LogsrvApi.Filesystem.list(:pages)
+    pages
+  end
+  def journals do
+    {:ok, journals} =  LogsrvApi.Filesystem.list(:journals)
+    journals
+  end
+
 end

logsrv/apps/logsrv_web/lib/logsrv_web/views/post_view.ex

@@ -3,7 +3,7 @@ defmodule LogsrvWeb.PostView do
   import LogsrvApi.Journal
 
   def post_path(post) do
-    "/post/#{post.filename}"
+    "/post/#{post.slug}"
   end
 
   def cleanup(str) do