Powerful Data Access in Clojure

Yannick Scherer

The Perfect Data Source

?
  • one endpoint
  • one shape ()
  • queriable dimensions ()
  • self-contained
  • no unnecessary data

The Perfect Data Source


(defn fetch-cubes!
  [datasource cube-query]
  ...)
            

(defn fetch-view-data!
  [datasource view]
  (->> (build-cube-query view)
       (fetch-cubes! datasource)
       (transform-for-view)))

Reality!

A Federation of Perfect Data Sources

  • one endpoint per shape
  • queriable dimensions
  • self-contained
  • no unnecessary data

(defn fetch-cubes!
  [datasource cube-query]
  ...)

(defn fetch-circles!
  [datasource circle-query]
  ...)

;; ...
            

(defn fetch-view-data!
  [datasources view]
  (let [cubes   (->> (build-cube-query view)
                     (fetch-cubes! (:cube-source datasources)))
        circles (->> (build-circle-query view)
                     (fetch-circles! (:circle-source datasources)))]
    (transform-for-view cubes circles)))

An Opportunity for Parallelisation


(defn fetch-view-data!
  [datasources view]
  (let [cubes   (->> (build-cube-query view)
                     (fetch-cubes! (:cube-source datasources))
                     (future))
        circles (->> (build-circle-query view)
                     (fetch-circles! (:circle-source datasources))
                     (future))]
    (transform-for-view @cubes @circles)))

More Reality!

Linked Data Sources

  • one endpoint per shape
  • queriable dimensions
  • explicit dependencies
  • no unnecessary data

(defn fetch-view-data!
  [datasources view]
  (let [floppies (->> (build-floppy-query view)
                      (fetch-floppies! (:floppy-source datasources)))
        gears    (->> (extract-gear-references floppies)
                      (build-gear-subquery view)
                      (fetch-gears! (:gear-source datasources)))]
    (transform-for-view
      (attach-gears floppies gears))))

More Levels


(defn fetch-view-data!
  [datasources view]
  (let [floppies ...
        gears    ...
        cubes    (->> (concat
                        (extract-floppy-cube-references floppies)
                        (extract-gear-cube-references gears))
                      (build-cube-subquery view)
                      (fetch-cubes! (:cube-source datasources)))]
    (transform-for-view
      (attach-gears
        (attach-floppy-cubes floppies cubes)
        (attach-gear-cubes gears cubes)))))

Variations of the same Shape


(defn fetch-view-data!
  [datasources view]
  (let [floppies ...
        gears    ...
        g-cubes  (->> (extract-floppy-cube-references floppies)
                      (build-floppy-cube-subquery view)
                      (fetch-cubes! (:cube-source datasources)))
        f-cubes  (->> (extract-gear-cube-references gears)
                      (build-gear-cube-subquery view)
                      (fetch-cubes! (:cube-source datasources)))]
    (transform-for-view
      (attach-gears
        (attach-floppy-cubes floppies f-cubes)
        (attach-gear-cubes gears g-cubes)))))
            

(defn fetch-view-data!
  [datasources view]
  (let [floppies ...
        gears    ...
        g-cubes  (->> (extract-floppy-cube-references floppies)
                      (build-floppy-cube-subquery view)
                      (fetch-cubes! (:cube-source datasources))
                      (future))
        f-cubes  (->> (extract-gear-cube-references gears)
                      (build-gear-cube-subquery view)
                      (fetch-cubes! (:cube-source datasources))
                      (future))]
    (transform-for-view
      (attach-gears
        (attach-floppy-cubes floppies @f-cubes)
        (attach-gear-cubes gears @g-cubes)))))

Things you don't want to care about

  • Reference Locations
  • Fetching Order/Parallelism
  • Redundant Fetches
  • Soundness

Haxl

Haxl

A Haskell library that simplifies access to remote data, such as databases or web-based services.

Haxl

  • Implicit concurrency using applicative functors.
  • Implicit caching.
commonFriendsOfFriends id1 id2
concat <$> mapM friendsOf
intersect <$> <*>
concat <$> mapM friendsOf
friendsOf id1
friendsOf id2

Request Definition


data FriendReq a where
  FriendsOf :: Id -> FriendReq [Id]
  deriving (Typeable)

instance DataSource u FriendReq where
  fetch _state _flags _userEnv blockedFetches =
    AsyncFetch $ \inner -> do ...

friendsOf :: Id -> Haxl [Id]
friendsOf id = dataFetch (FriendsOf id)

commonFriendsOfFriends id1 id2 = do
  friends1 <- friendsOf id1
  friends2 <- friendsOf id2
  fofs1    <- concat <$> mapM friendsOf friends1
  fofs2    <- concat <$> mapM friendsOf friends2
  intersect fofs1 fofs2

Request Execution


main = do
  env  <- ...
  id1  <- ...
  id2  <- ...
  fofs <- runHaxl env $ commonFriendsOfFriends id1 id2
  print fofs

Muse & Urania

Muse

A Clojure library that works hard to make your relationship with remote data simple & enjoyable.

Muse

  • Implements functors using the cats library.
  • Leverages protocols and records to declare data sources.
  • Uses core.async channels for concurrency.

Data Sources


(defrecord FriendsOf [id]
  muse/DataSource
  (fetch [_]
    (go
      (set (fetch-friend-ids! id)))))

Composition (fmap)


(muse/fmap count (->FriendsOf id1))
(muse/fmap set/intersection (->FriendsOf id1) (->FriendsOf id2))

Bind (flat-map)


(->> (->FriendsOf id1)
     (muse/fmap first)
     (muse/flat-map ->FriendsOf))

Map (traverse)


(defn friends-of-friends
  [id]
  (->> (->FriendsOf id)
       (muse/traverse ->FriendsOf)
       (muse/fmap #(reduce set/union %))))

The Good


(defn common-friends-of-friends
  [id1 id2]
  (muse/fmap
    set/intersection
    (friends-of-friends id1)
    (friends-of-friends id2)))

(muse/run!! (common-friends-of-friends 1 2))
;; => #{3 4 5}

The Batch


(defrecord FriendsOf [id]
  muse/DataSource
  (fetch [_]
    (go
      (set (fetch-friend-ids! id))))

  muse/BatchedSource
  (fetch-multi [_ users]
    (let [ids (cons id (map :id users))]
      (->> ids ...))))

The Ugly?


(defn user-and-friends-by-name
  [name]
  (muse/flat-map
    (fn [{:keys [id] :as user}]
      (muse/fmap
        #(assoc user :friends %)
        (->FriendsOf id)))
    (->User name)))

Urania

  • Fork of Muse.
  • Promises instead of channels.
  • fmap map
  • flat-map mapcat
  • Allows passing of environment to fetch function.

Things you don't want to care about

  • Reference Locations
  • Fetching Order/Parallelism
  • Redundant Fetches
  • Soundness

GraphQL

GraphQL

Describe your data.

Ask for what you want.

Get predictable results.


{
  commonFriends(id1: 1, id2: 2) {
    name,
    address { city }
  }
}
            

{
  "commonFriends": [
    { "name": "Nobody", "address": { "city": "Nowhere" }},
    ...
  ]
}
            

{
  commonFriends(id1: 1, id2: 2) {
    name,
    address { city },
    friends { friends { friends { name } } }
  }
}
            

{
  "commonFriends": [
    {
      "name": "Nobody",
      "address": { "city": "Nowhere" },
      "friends": [{ "friends": [{ "friends": [{ "name": "Who" }]}]}]
    },
    ...
  ]
}

claro

claro

A library that allows you to streamline your data access, providing powerful optimisations and abstractions along the way.

Any datastructure is a datasource

Any datastructure is a datasource:


(engine/run!!
  {:friends1 (->FriendsOf 1)
   :friends2 (->FriendsOf 2)})
;; => {:friends1 #{2 3}, :friends2 #{2 5}}

Compare:


(muse/run!!
  (muse/fmap
    #(hash-map :friends1 %1 :friends2 %2)
    (->FriendsOf 1)
    (->FriendsOf 2)))
;; => {:friends1 #{2 3}, :friends2 #{2 5}}

Datasources can produce more datasources

Datasources can produce more datasources:


(defrecord User [id]
  data/Resolvable
  (resolve! [_ env]
    (d/future
      (when-let [user (fetch-user! (:db env) id)]
        (assoc user :friends (->FriendsOf (:id user)))))))

fewer (fmap ...) calls
unnecessary data‽

Datasources can produce infinite trees

Datasources can produce infinite trees:


(defrecord FriendsOf [id]
  data/Resolvable
  (resolve! [_ env]
    (d/future
      (let [friends (fetch-friend-ids! (:db env) id)]
        (set (map #(->User %) friends))))))
            

(engine/run!! (->User 1))
;; => IllegalStateException: maximum batch count exceeded (33/32).

Tree Projections

Selection


{:id      projection/leaf
 :name    projection/leaf
 :friends [{:name projection/leaf}]}
            

(engine/run!! (projection/apply (->User 1) ...))
;; => {:id      1,
;;     :name    "Someone",
;;     :friends [{:name "Nobody"} {:name "NobodyElse"}]}

Merge


(projection/union
  base-user
  {:friends [(projection/extract :name)]})
            

(engine/run!! (projection/apply (->User 1) ...))
;; => {:id      1,
;;     :name    "Someone",
;;     :friends ["Nobody" "NobodyElse"]}

tree projections are composable!

Transform


{:id      projection/leaf
 :name    projection/leaf
 :friends (projection/transform count [{}])}
            

(engine/run!! (projection/apply (->User 1) ...))
;; => {:id      1,
;;     :name    "Someone",
;;     :friends 2}

Claro Que Sì

  • Create one rich, reusable, infinite tree of data. 
    
(def Root
  {:user  (map->User {})
   :users (map->Users {})
   ...})
  • Select/transform the parts you need ad-hoc. 
    
(defn fetch-view-data!
  [view]
  (->> (build-projection view)
       (projection/apply Root)
       (engine/run!!)))
?

The Perfect Data Source

Thank you!