diff --git a/evil-collection-ivy.el b/evil-collection-ivy.el
index 000562e..c4d0bb6 100644
--- a/evil-collection-ivy.el
+++ b/evil-collection-ivy.el
@@ -75,7 +75,23 @@
     "gc" 'ivy-occur-toggle-calling
 
     ;; quit
-    "q" 'quit-window))
+    "q" 'quit-window)
+
+  (defvar evil-collection-setup-minibuffer)
+  (when evil-collection-setup-minibuffer
+    (evil-define-key 'normal ivy-minibuffer-map
+      (kbd "<escape>") 'abort-recursive-edit
+      (kbd "<return>") 'exit-minibuffer
+      (kbd "C-m") 'ivy-done
+      "j" 'ivy-next-line
+      "k" 'ivy-previous-line)
+
+    (evil-define-key 'insert ivy-minibuffer-map
+      [backspace] 'ivy-backward-delete-char
+      (kbd "C-r") 'ivy-reverse-i-search
+      (kbd "C-n") 'ivy-next-line
+      (kbd "C-p") 'ivy-previous-line)))
+
 
 (provide 'evil-collection-ivy)
 ;;; evil-collection-ivy.el ends here
diff --git a/evil-collection-minibuffer.el b/evil-collection-minibuffer.el
index 05d368e..1536d84 100644
--- a/evil-collection-minibuffer.el
+++ b/evil-collection-minibuffer.el
@@ -59,11 +59,6 @@ it does not have a mode."
     (evil-define-key* 'normal map (kbd "<escape>") 'abort-recursive-edit)
     (evil-define-key* 'normal map (kbd "<return>") 'exit-minibuffer))
 
-  (with-eval-after-load 'ivy
-    (defvar ivy-minibuffer-map)
-    (evil-define-key* 'normal ivy-minibuffer-map (kbd "<escape>") 'abort-recursive-edit)
-    (evil-define-key* 'normal ivy-minibuffer-map (kbd "<return>") 'exit-minibuffer))
-
   (add-hook 'minibuffer-setup-hook 'evil-collection-minibuffer-insert)
   ;; Because of the above minibuffer-setup-hook, some evil-ex bindings need be reset.
   (evil-define-key 'normal evil-ex-completion-map (kbd "<escape>") 'abort-recursive-edit)