drm/nouveau/instmem: completely new implementation, as a subdev module

v2 (Ben Skeggs):
- some fixes for 64KiB PAGE_SIZE
- fix porting issues in (currently unused) nv41/nv44 pciegart code

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

69 files changed, 2777 insertions, 2169 deletions

diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index 68043a40d9e..9a86ae3e680 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -9,6 +9,7 @@ ccflags-y += -I$(src)
 nouveau-y := core/core/client.o
 nouveau-y += core/core/engine.o
 nouveau-y += core/core/enum.o
+nouveau-y += core/core/gpuobj.o
 nouveau-y += core/core/handle.o
 nouveau-y += core/core/mm.o
 nouveau-y += core/core/namedb.o
@@ -19,6 +20,9 @@ nouveau-y += core/core/printk.o
 nouveau-y += core/core/ramht.o
 nouveau-y += core/core/subdev.o
 
+nouveau-y += core/subdev/bar/base.o
+nouveau-y += core/subdev/bar/nv50.o
+nouveau-y += core/subdev/bar/nvc0.o
 nouveau-y += core/subdev/bios/base.o
 nouveau-y += core/subdev/bios/bit.o
 nouveau-y += core/subdev/bios/conn.o
@@ -66,10 +70,10 @@ nouveau-y += core/subdev/gpio/nvd0.o
 nouveau-y += core/subdev/i2c/base.o
 nouveau-y += core/subdev/i2c/aux.o
 nouveau-y += core/subdev/i2c/bit.o
+nouveau-y += core/subdev/instmem/base.o
 nouveau-y += core/subdev/instmem/nv04.o
 nouveau-y += core/subdev/instmem/nv40.o
 nouveau-y += core/subdev/instmem/nv50.o
-nouveau-y += core/subdev/instmem/nvc0.o
 nouveau-y += core/subdev/ltcg/nvc0.o
 nouveau-y += core/subdev/mc/base.o
 nouveau-y += core/subdev/mc/nv04.o
@@ -80,6 +84,9 @@ nouveau-y += core/subdev/mc/nvc0.o
 nouveau-y += core/subdev/timer/base.o
 nouveau-y += core/subdev/timer/nv04.o
 nouveau-y += core/subdev/vm/base.o
+nouveau-y += core/subdev/vm/nv04.o
+nouveau-y += core/subdev/vm/nv41.o
+nouveau-y += core/subdev/vm/nv44.o
 nouveau-y += core/subdev/vm/nv50.o
 nouveau-y += core/subdev/vm/nvc0.o
 
diff --git a/drivers/gpu/drm/nouveau/core/core/ramht.c b/drivers/gpu/drm/nouveau/core/core/ramht.c
index 59c16192f0e..5c22864fbd2 100644
--- a/drivers/gpu/drm/nouveau/core/core/ramht.c
+++ b/drivers/gpu/drm/nouveau/core/core/ramht.c
@@ -86,7 +86,6 @@ nouveau_ramht_insert(struct nouveau_channel *chan, u32 handle,
 {
 	struct drm_device *dev = chan->dev;
 	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 	struct nouveau_ramht_entry *entry;
 	struct nouveau_gpuobj *ramht = chan->ramht->gpuobj;
 	unsigned long flags;
@@ -104,21 +103,21 @@ nouveau_ramht_insert(struct nouveau_channel *chan, u32 handle,
 	nouveau_gpuobj_ref(gpuobj, &entry->gpuobj);
 
 	if (dev_priv->card_type < NV_40) {
-		ctx = NV_RAMHT_CONTEXT_VALID | (gpuobj->pinst >> 4) |
+		ctx = NV_RAMHT_CONTEXT_VALID | (gpuobj->addr >> 4) |
 		      (chan->id << NV_RAMHT_CONTEXT_CHANNEL_SHIFT) |
 		      (gpuobj->engine << NV_RAMHT_CONTEXT_ENGINE_SHIFT);
 	} else
 	if (dev_priv->card_type < NV_50) {
-		ctx = (gpuobj->pinst >> 4) |
+		ctx = (gpuobj->addr >> 4) |
 		      (chan->id << NV40_RAMHT_CONTEXT_CHANNEL_SHIFT) |
 		      (gpuobj->engine << NV40_RAMHT_CONTEXT_ENGINE_SHIFT);
 	} else {
 		if (gpuobj->engine == NVOBJ_ENGINE_DISPLAY) {
-			ctx = (gpuobj->cinst << 10) |
+			ctx = (gpuobj->node->offset << 10) |
 			      (chan->id << 28) |
 			      chan->id; /* HASH_TAG */
 		} else {
-			ctx = (gpuobj->cinst >> 4) |
+			ctx = (gpuobj->node->offset >> 4) |
 			      ((gpuobj->engine <<
 				NV40_RAMHT_CONTEXT_ENGINE_SHIFT));
 		}
@@ -137,7 +136,7 @@ nouveau_ramht_insert(struct nouveau_channel *chan, u32 handle,
 			nv_wo32(ramht, co + 4, ctx);
 
 			spin_unlock_irqrestore(&chan->ramht->lock, flags);
-			instmem->flush(dev);
+			nvimem_flush(dev);
 			return 0;
 		}
 		NV_DEBUG(dev, "collision ch%d 0x%08x: h=0x%08x\n",
@@ -184,8 +183,6 @@ static void
 nouveau_ramht_remove_hash(struct nouveau_channel *chan, u32 handle)
 {
 	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nouveau_instmem_engine *instmem = &dev_priv->engine.instmem;
 	struct nouveau_gpuobj *ramht = chan->ramht->gpuobj;
 	unsigned long flags;
 	u32 co, ho;
@@ -201,7 +198,7 @@ nouveau_ramht_remove_hash(struct nouveau_channel *chan, u32 handle)
 				 chan->id, co, handle, nv_ro32(ramht, co + 4));
 			nv_wo32(ramht, co + 0, 0x00000000);
 			nv_wo32(ramht, co + 4, 0x00000000);
-			instmem->flush(dev);
+			nvimem_flush(dev);
 			goto out;
 		}
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/bsp/nv84.c b/drivers/gpu/drm/nouveau/core/engine/bsp/nv84.c
index db3b5736983..4b809319e83 100644
--- a/drivers/gpu/drm/nouveau/core/engine/bsp/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/bsp/nv84.c
@@ -25,7 +25,6 @@
 #include "drmP.h"
 #include "nouveau_drv.h"
 #include "nouveau_util.h"
-#include <subdev/vm.h>
 #include <core/ramht.h>
 
 /*XXX: This stub is currently used on NV98+ also, as soon as this becomes
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c b/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
index fec52463456..9150c5ed16c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nva3.c
@@ -26,7 +26,6 @@
 #include "drmP.h"
 #include "nouveau_drv.h"
 #include "nouveau_util.h"
-#include <subdev/vm.h>
 #include <core/ramht.h>
 #include "fuc/nva3.fuc.h"
 
@@ -38,7 +37,6 @@ static int
 nva3_copy_context_new(struct nouveau_channel *chan, int engine)
 {
 	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nouveau_gpuobj *ramin = chan->ramin;
 	struct nouveau_gpuobj *ctx = NULL;
 	int ret;
@@ -51,14 +49,14 @@ nva3_copy_context_new(struct nouveau_channel *chan, int engine)
 		return ret;
 
 	nv_wo32(ramin, 0xc0, 0x00190000);
-	nv_wo32(ramin, 0xc4, ctx->vinst + ctx->size - 1);
-	nv_wo32(ramin, 0xc8, ctx->vinst);
+	nv_wo32(ramin, 0xc4, ctx->addr + ctx->size - 1);
+	nv_wo32(ramin, 0xc8, ctx->addr);
 	nv_wo32(ramin, 0xcc, 0x00000000);
 	nv_wo32(ramin, 0xd0, 0x00000000);
 	nv_wo32(ramin, 0xd4, 0x00000000);
-	dev_priv->engine.instmem.flush(dev);
+	nvimem_flush(dev);
 
-	atomic_inc(&chan->vm->engref[engine]);
+	nvvm_engref(chan->vm, engine, 1);
 	chan->engctx[engine] = ctx;
 	return 0;
 }
@@ -84,7 +82,7 @@ nva3_copy_context_del(struct nouveau_channel *chan, int engine)
 	for (i = 0xc0; i <= 0xd4; i += 4)
 		nv_wo32(chan->ramin, i, 0x00000000);
 
-	atomic_dec(&chan->vm->engref[engine]);
+	nvvm_engref(chan->vm, engine, -1);
 	nouveau_gpuobj_ref(NULL, &ctx);
 	chan->engctx[engine] = ctx;
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
index 926f21c0ebc..f39de5a593d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/copy/nvc0.c
@@ -26,7 +26,6 @@
 #include "drmP.h"
 #include "nouveau_drv.h"
 #include "nouveau_util.h"
-#include <subdev/vm.h>
 #include <core/ramht.h>
 #include "fuc/nvc0.fuc.h"
 
@@ -49,7 +48,6 @@ nvc0_copy_context_new(struct nouveau_channel *chan, int engine)
 	struct nvc0_copy_engine *pcopy = nv_engine(chan->dev, engine);
 	struct nvc0_copy_chan *cctx;
 	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nouveau_gpuobj *ramin = chan->ramin;
 	int ret;
 
@@ -62,14 +60,14 @@ nvc0_copy_context_new(struct nouveau_channel *chan, int engine)
 	if (ret)
 		return ret;
 
-	ret = nouveau_gpuobj_map_vm(cctx->mem, NV_MEM_ACCESS_RW, chan->vm,
+	ret = nouveau_gpuobj_map_vm(cctx->mem, chan->vm, NV_MEM_ACCESS_RW,
 				   &cctx->vma);
 	if (ret)
 		return ret;
 
 	nv_wo32(ramin, pcopy->ctx + 0, lower_32_bits(cctx->vma.offset));
 	nv_wo32(ramin, pcopy->ctx + 4, upper_32_bits(cctx->vma.offset));
-	dev_priv->engine.instmem.flush(dev);
+	nvimem_flush(dev);
 	return 0;
 }
 
@@ -88,7 +86,7 @@ nvc0_copy_context_del(struct nouveau_channel *chan, int engine)
 	struct drm_device *dev = chan->dev;
 	u32 inst;
 
-	inst  = (chan->ramin->vinst >> 12);
+	inst  = (chan->ramin->addr >> 12);
 	inst |= 0x40000000;
 
 	/* disable fifo access */
diff --git a/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c b/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
index 388138946ad..63051ab0ecc 100644
--- a/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/nv84.c
@@ -25,7 +25,6 @@
 #include "drmP.h"
 #include "nouveau_drv.h"
 #include "nouveau_util.h"
-#include <subdev/vm.h>
 #include <core/ramht.h>
 
 struct nv84_crypt_engine {
@@ -36,7 +35,6 @@ static int
 nv84_crypt_context_new(struct nouveau_channel *chan, int engine)
 {
 	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nouveau_gpuobj *ramin = chan->ramin;
 	struct nouveau_gpuobj *ctx;
 	int ret;
@@ -49,14 +47,14 @@ nv84_crypt_context_new(struct nouveau_channel *chan, int engine)
 		return ret;
 
 	nv_wo32(ramin, 0xa0, 0x00190000);
-	nv_wo32(ramin, 0xa4, ctx->vinst + ctx->size - 1);
-	nv_wo32(ramin, 0xa8, ctx->vinst);
+	nv_wo32(ramin, 0xa4, ctx->addr + ctx->size - 1);
+	nv_wo32(ramin, 0xa8, ctx->addr);
 	nv_wo32(ramin, 0xac, 0);
 	nv_wo32(ramin, 0xb0, 0);
 	nv_wo32(ramin, 0xb4, 0);
-	dev_priv->engine.instmem.flush(dev);
+	nvimem_flush(dev);
 
-	atomic_inc(&chan->vm->engref[engine]);
+	nvvm_engref(chan->vm, engine, 1);
 	chan->engctx[engine] = ctx;
 	return 0;
 }
@@ -68,7 +66,7 @@ nv84_crypt_context_del(struct nouveau_channel *chan, int engine)
 	struct drm_device *dev = chan->dev;
 	u32 inst;
 
-	inst  = (chan->ramin->vinst >> 12);
+	inst  = (chan->ramin->addr >> 12);
 	inst |= 0x80000000;
 
 	/* mark context as invalid if still on the hardware, not
@@ -84,7 +82,7 @@ nv84_crypt_context_del(struct nouveau_channel *chan, int engine)
 
 	nouveau_gpuobj_ref(NULL, &ctx);
 
-	atomic_dec(&chan->vm->engref[engine]);
+	nvvm_engref(chan->vm, engine, -1);
 	chan->engctx[engine] = NULL;
 }
 
@@ -93,7 +91,6 @@ nv84_crypt_object_new(struct nouveau_channel *chan, int engine,
 		      u32 handle, u16 class)
 {
 	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nouveau_gpuobj *obj = NULL;
 	int ret;
 
@@ -104,7 +101,7 @@ nv84_crypt_object_new(struct nouveau_channel *chan, int engine,
 	obj->class  = class;
 
 	nv_wo32(obj, 0x00, class);
-	dev_priv->engine.instmem.flush(dev);
+	nvimem_flush(dev);
 
 	ret = nouveau_ramht_insert(chan, handle, obj);
 	nouveau_gpuobj_ref(NULL, &obj);
diff --git a/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c b/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
index 563f60d1ce6..c9adc1b8a7d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
+++ b/drivers/gpu/drm/nouveau/core/engine/crypt/nv98.c
@@ -26,7 +26,6 @@
 
 #include "nouveau_drv.h"
 #include "nouveau_util.h"
-#include <subdev/vm.h>
 #include <core/ramht.h>
 
 #include "fuc/nv98.fuc.h"
@@ -43,7 +42,6 @@ static int
 nv98_crypt_context_new(struct nouveau_channel *chan, int engine)
 {
 	struct drm_device *dev = chan->dev;
-	struct drm_nouveau_private *dev_priv = dev->dev_private;
 	struct nv98_crypt_priv *priv = nv_engine(dev, engine);
 	struct nv98_crypt_chan *cctx;
 	int ret;
@@ -52,7 +50,7 @@ nv98_crypt_context_new(struct nouveau_channel *chan, int engine)
 	if (!cctx)
 		return -ENOMEM;
 
-	atomic_inc(&chan->vm->engref[engine]);
+	nvvm_engref(chan->vm, engine, 1);
 
 	ret = nouveau_gpuobj_new(dev, chan, 256, 0, NVOBJ_FLAG_ZERO_ALLOC |
 				 NVOBJ_FLAG_ZERO_FREE, &cctx->mem);
@@ -60,12 +58,12 @@ nv98_crypt_context_new(struct nouveau_channel *chan, int engine)
 		goto error;
 
 	nv_wo32(chan->ramin, 0xa0, 0x00190000);
-	nv_wo32(chan->ramin, 0xa4, cctx->mem->vinst + cctx->mem->size - 1);
-	nv_wo32(chan->ramin, 0xa8, cctx->mem->vinst);
+	nv_wo32(chan->ramin, 0xa4, cctx->mem->addr + cctx->mem->size - 1);
+	nv_wo32(chan->ramin, 0xa8, cctx->mem->addr);
 	nv_wo32(chan->ramin, 0xac, 0x00000000);
 	nv_wo32(chan->ramin, 0xb0, 0x00000000);
 	nv_wo32(chan->ramin, 0xb4, 0x00000000);
-	dev_priv->engine.instmem.flush(dev);
+	nvimem_flush(dev);
 
 error:
 	if (ret)
@@ -84,7 +82,7 @@ nv98_crypt_context_del(struct nouveau_channel *chan, int engine)
 
 	nouveau_gpuobj_ref(NULL, &cctx->mem);
 
-	atomic_dec(&chan->vm->engref[engine]);
+	nvvm_engref(chan->vm, engine, -1);
 	chan->engctx[engine] = NULL;
 	kfree(cctx);
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
index ba76cf094a6..6ab7eb0dd9b 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
@@ -32,8 +32,6 @@
 #include <core/ramht.h>
 #include "nouveau_software.h"
 
-#include <core/subdev/instmem/nv04.h>
-
 static struct ramfc_desc {
 	unsigned bits:6;
 	unsigned ctxs:5;
@@ -120,7 +118,7 @@ nv04_fifo_context_new(struct nouveau_channel *chan, int engine)
 	/* initialise default fifo context */
 	nv_wo32(priv->ramfc, fctx->ramfc + 0x00, chan->pushbuf_base);
 	nv_wo32(priv->ramfc, fctx->ramfc + 0x04, chan->pushbuf_base);
-	nv_wo32(priv->ramfc, fctx->ramfc + 0x08, chan->pushbuf->pinst >> 4);
+	nv_wo32(priv->ramfc, fctx->ramfc + 0x08, chan->pushbuf->addr >> 4);
 	nv_wo32(priv->ramfc, fctx->ramfc + 0x10,
 			     NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
 			     NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
@@ -203,9 +201,9 @@ nv04_fifo_init(struct drm_device *dev, int engine)
 
 	nv_wr32(dev, NV03_PFIFO_RAMHT, (0x03 << 24) /* search 128 */ |
 				       ((dev_priv->ramht->bits - 9) << 16) |
-				       (dev_priv->ramht->gpuobj->pinst >> 8));
-	nv_wr32(dev, NV03_PFIFO_RAMRO, priv->ramro->pinst >> 8);
-	nv_wr32(dev, NV03_PFIFO_RAMFC, priv->ramfc->pinst >> 8);
+				       (dev_priv->ramht->gpuobj->addr >> 8));
+	nv_wr32(dev, NV03_PFIFO_RAMRO, priv->ramro->addr >> 8);
+	nv_wr32(dev, NV03_PFIFO_RAMFC, priv->ramfc->addr >> 8);
 
 	nv_wr32(dev, NV03_PFIFO_CACHE1_PUSH1, priv->base.channels);
 
@@ -486,15 +484,14 @@ int
 nv04_fifo_create(struct drm_device *dev)
 {
 	struct drm_nouveau_private *dev_priv = dev->dev_private;
-	struct nv04_instmem_priv *imem = dev_priv->engine.instmem.priv;
 	struct nv04_fifo_priv *priv;
 
 	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 	if (!priv)
 		return -ENOMEM;
 
-	nouveau_gpuobj_ref(imem->ramro, &priv->ramro);
-	nouveau_gpuobj_ref(imem->ramfc, &priv->ramfc);
+	nouveau_gpuobj_ref(nvimem_ramro(dev), &priv->ramro);
+	nouveau_gpuobj_ref(nvimem_ramfc(dev), &priv->ramfc);
 
 	priv->base.base.destroy = nv04_fifo_destroy;
 	priv->base.base.init = nv04_fifo_init;
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
index 0da287caf43..2d38fa88f9c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
@@ -31,8 +31,6 @@
 #include "nouveau_util.h"
 #include <core/ramht.h>
 
-#include <core/subdev/instmem/nv04.h>
-
 static struct ramfc_desc {
 	unsigned bits:6;
 	unsigned ctxs:5;
@@ -91,7 +89,7 @@ nv10_fifo_context_new(struct nouveau_channel *chan, int engine)
 	/* initialise default fifo context */
 	nv_wo32(priv->ramfc, fctx->ramfc + 0x00, chan->pushbuf_base);
 	nv_wo32(priv->ramfc, fctx->ramfc + 0x04, chan->pushbuf_base);
-	nv_wo32(priv->ramfc, fctx->ramfc + 0x0c, chan->pushbuf->pinst >> 4);
+	nv_wo32(priv->ramfc, fctx->ramfc + 0x0c, chan->pushbuf->addr >> 4);
 	nv_wo32(priv->ramfc, fctx->ramfc + 0x14,
 			     NV_PFIFO_CACHE1_DMA_FETCH_TRIG_128_BYTES |
 			     NV_PFIFO_CACHE1_DMA_FETCH_SIZE_128_BYTES |
@@ -115,15 +113,14 @@ int
 nv10_fifo_create(struct drm_device *dev)
 {
 	struct drm_nouveau_private *dev_priv = dev->dev_private;