c# – 将SurfaceTexture渲染为Unity Texture2D

所以我要做的是将SurfaceTexture从Android插件渲染到Unity Texture2D.

这是我的Unity代码：

public class AndroidHandler : MonoBehaviour {

    [SerializeField]
    private RawImage _rawImage;

    private Texture2D _inputTexture;
    private AndroidJavaObject androidStreamerObj;
    private System.IntPtr _nativePtr;

    void Start () {

        _rawImage.material.SetTextureScale("_MainTex",new Vector2(-1,-1));
        InitAndroidStreamerObject();
    }

    private void InitAndroidStreamerObject()
    {
        androidStreamerObj = new AndroidJavaObject("makeitbetter.figazzz.com.vitamiousing7.AndroidStreamer");

        Int32 texPtr = androidStreamerObj.Call <Int32> ("GetTexturePtr");
        Debug.Log("texture pointer? " + texPtr);
        Texture2D nativeTexture = Texture2D.CreateExternalTexture (128,128,TextureFormat.RGBA32,false,new System.IntPtr(texPtr));

        _rawImage.texture = nativeTexture;
    }

    public void StartStream()
    {
        string streamLink = "rtmp://live.hkstv.hk.lxdns.com/live/hks"; //"rtsp://wowzaec2demo.streamlock.net/vod/mp4:BigBuckBunny_115k.mov"; //"rtmp://live.hkstv.hk.lxdns.com/live/hks";
        androidStreamerObj.Call("LaunchStream",streamLink);
    }

    void Update()
    {
        androidStreamerObj.Call("DrawFrame");
    }
}

你可以看到它非常基本.我要求我的Android插件创建openGLTexture,我正在使用全新纹理的指针在Unity中分配Texture2D.

然后是我的Android插件代码：

public class AndroidStreamer {

    private final int FLOAT_SIZE_BYTES = 4;
    private final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
    private final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
    private final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;

    private Activity _currActivity;
    private VideoView _streamConnection;

    private Surface _cachedSurface;
    private SurfaceTexture _cachedSurfaceTexture;

    private Boolean isNewFrame = false;

    //open gl
    private int texWidth = 128;
    private int texHeight = 128;
    private float[] mMVPMatrix = new float[16];
    private float[] mSTMatrix = new float[16];

    private int glProgram;
    private int muMVPMatrixHandle;
    private int muSTMatrixHandle;
    private int maPositionHandle;
    private int maTextureHandle;
    private int unityTextureID = -1;
    private int mTextureId = -1;   //surface texture id
    private int idFBO = -1;
    private int idRBO = -1;

    private final float[] mTriangleVerticesData = {
            // X,Y,Z,U,V
            -1.0f,-1.0f,0.f,1.0f,1.f,};

    private FloatBuffer mTriangleVertices;

    private final String vertexShaderCode =
            "uniform mat4 uMVPMatrix;n" +
                    "uniform mat4 uSTMatrix;n" +
                    "attribute vec4 aPosition;n" +
                    "attribute vec4 aTextureCoord;n" +
                    "varying vec2 vTextureCoord;n" +
                    "void main() {n" +
                    "    gl_Position = uMVPMatrix * aPosition;n" +
                    "    vTextureCoord = (uSTMatrix * aTextureCoord).xy;n" +
                    "}n";

    private final String fragmentShaderCode =
            "#extension GL_OES_EGL_image_external : requiren" +
                    "precision mediump float;n" +      // highp here doesn't seem to matter
                    "varying vec2 vTextureCoord;n" +
                    "uniform samplerExternalOES sTexture;n" +
                    "void main() {n" +
                    "    gl_FragColor = texture2D(sTexture,vTextureCoord);n" +
                    "}n";

    public AndroidStreamer() {
        Log.d("Unity","AndroidStreamer was initialized");

        _currActivity = UnityPlayer.currentActivity;
        Vitamio.isInitialized(_currActivity);

        _currActivity.runOnUiThread(new Runnable() {
            @Override
            public void run() {
                _streamConnection = new VideoView(_currActivity);
                _currActivity.addContentView(_streamConnection,new FrameLayout.LayoutParams(100,100));
            }
        });

        mTriangleVertices = ByteBuffer.allocateDirect(
                mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
                .order(ByteOrder.nativeOrder()).asFloatBuffer();
        mTriangleVertices.put(mTriangleVerticesData).position(0);
        Matrix.setIdentityM(mSTMatrix,0);

        initShaderProgram();
    }

    private void initShaderProgram()
    {
        Log.d("Unity","initShaderProgram");
        int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER,vertexShaderCode);
        int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER,fragmentShaderCode);

        glProgram = GLES20.glCreateProgram();
        GLES20.glAttachShader(glProgram,vertexShader);
        checkGlError("glAttachVertexShader");
        GLES20.glAttachShader(glProgram,fragmentShader);
        checkGlError("glAttachFragmentShader");
        GLES20.glLinkProgram(glProgram);
        checkGlError("glLinkProgram");

        maPositionHandle = GLES20.glGetAttribLocation(glProgram,"aPosition");
        checkLocation(maPositionHandle,"aPosition");
        maTextureHandle = GLES20.glGetAttribLocation(glProgram,"aTextureCoord");
        checkLocation(maTextureHandle,"aTextureCoord");

        muMVPMatrixHandle = GLES20.glGetUniformLocation(glProgram,"uMVPMatrix");
        checkLocation(muMVPMatrixHandle,"uVMPMatrix");
        muSTMatrixHandle = GLES20.glGetUniformLocation(glProgram,"uSTMatrix");
        checkLocation(muSTMatrixHandle,"uSTMatrix");
    }

    private int loadShader(int shaderType,String source) {
        int shader = GLES20.glCreateShader(shaderType);
        if (shader != 0) {
            GLES20.glShaderSource(shader,source);
            GLES20.glCompileShader(shader);
            int[] compiled = new int[1];
            GLES20.glGetShaderiv(shader,GLES20.GL_COMPILE_STATUS,compiled,0);
            if (compiled[0] == 0) {
                Log.e("Unity","Could not compile shader " + shaderType + ":");
                Log.e("Unity",GLES20.glGetShaderInfoLog(shader));
                GLES20.glDeleteShader(shader);
                shader = 0;
            }
        }
        return shader;
    }

    private void checkLocation(int location,String label) {
        if (location < 0) {
            throw new RuntimeException("Unable to locate '" + label + "' in program");
        }
    }

    private void checkGlError(String op) {
        int error;
        while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
            Log.e("Unity",op + ": glError " + error);
            throw new RuntimeException(op + ": glError " + error);
        }
    }

    private void checkFrameBufferStatus()
    {
        int status = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER);
        checkGlError("glCheckFramebufferStatus");
        switch (status)
        {
            case GLES20.GL_FRAMEBUFFER_COMPLETE:
                Log.d("Unity","complete");
                break;
            case GLES20.GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT:
                Log.e("Unity","incomplete attachment");
                break;
            case GLES20.GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT:
                Log.e("Unity","incomplete missing attachment");
                break;
            case GLES20.GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS:
                Log.e("Unity","incomplete dimensions");
                break;
            case GLES20.GL_FRAMEBUFFER_UNSUPPORTED:
                Log.e("Unity","framebuffer unsupported");
                break;

            default : Log.d("Unity","default");
        }
    }

    private void initGLTexture()
    {
        Log.d("Unity","initGLTexture");
        int textures[] = new int[1];
        GLES20.glGenTextures(1,textures,0);
        checkGlError("glGenTextures initGLTexture");
        mTextureId = textures[0];

        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        checkGlError("glActiveTexture initGLTexture");
        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,mTextureId);
        checkGlError("glBindTexture initGLTexture");

        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,GLES20.GL_TEXTURE_MIN_FILTER,GLES20.GL_NEAREST);
        checkGlError("glTexParameterf initGLTexture");
        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,GLES20.GL_TEXTURE_MAG_FILTER,GLES20.GL_LINEAR);
        checkGlError("glTexParameterf initGLTexture");
    }

    public int GetTexturePtr()
    {
        Bitmap bitmap = Bitmap.createBitmap(texWidth,texHeight,Bitmap.Config.ARGB_8888);
        for(int x = 0; x < texWidth; x++)
        {
            for (int y = 0; y < texHeight; y++)
            {
                bitmap.setPixel(x,y,Color.argb(155,255,50,255));
            }
        }
        Log.d("Unity","Bitmap is: " + bitmap);

        ByteBuffer buffer = ByteBuffer.allocate(bitmap.getByteCount());
        bitmap.copyPixelsToBuffer(buffer);

        //GLES20.glEnable(GLES11Ext.GL_TEXTURE_EXTERNAL_OES);
        //checkGlError("glEnable GetTexturePtr");

        int textures[] = new int[1];
        GLES20.glGenTextures(1,0);
        checkGlError("0");
        unityTextureID = textures[0];

        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
        checkGlError("1");
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D,unityTextureID);
        checkGlError("2");

        GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D,GLES20.GL_RGBA,texWidth,GLES20.GL_UNSIGNED_BYTE,null);
        checkGlError("12");
        //GLUtils.texImage2D(GLES20.GL_TEXTURE_2D,bitmap,0);
        //checkGlError("3");
        GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,GLES20.GL_NEAREST);
        checkGlError("4");
        GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D,GLES20.GL_LINEAR);
        checkGlError("5");
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D,GLES20.GL_TEXTURE_WRAP_S,GLES20.GL_CLAMP_TO_EDGE);
        checkGlError("6");
        GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D,GLES20.GL_TEXTURE_WRAP_T,GLES20.GL_CLAMP_TO_EDGE);
        checkGlError("7");
        GLES20.glBindTexture(GLES20.GL_TEXTURE_2D,0);
        checkGlError("8");

        setupBuffers();
        Log.d("Unity","texture id returned: " + unityTextureID);

        return unityTextureID;
    }

    private void setupBuffers()
    {
        Log.d("Unity","setupBuffers");

        //framebuffer
        int buffers[] = new int[1];
        GLES20.glGenFramebuffers(1,buffers,0);
        checkGlError("9");
        idFBO = buffers[0];
        GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER,idFBO);
        checkGlError("10");

        //render buffer
        int rbuffers[] = new int[1];
        GLES20.glGenRenderbuffers(1,rbuffers,0);
        checkGlError("glGenRenderBuffers setupBuffers");
        idRBO = rbuffers[0];
        GLES20.glBindRenderbuffer(GLES20.GL_RENDERBUFFER,idRBO);
        checkGlError("glBindRenderBuffer setupBuffers");
        GLES20.glRenderbufferStorage(GLES20.GL_RENDERBUFFER,GLES20.GL_RGBA4,texHeight);
        checkGlError("glRenderBufferStorage setupBuffers");

        GLES20.glFramebufferRenderbuffer(GLES20.GL_FRAMEBUFFER,GLES20.GL_COLOR_ATTACHMENT0,GLES20.GL_RENDERBUFFER,idRBO);
        checkGlError("glFramebufferRenderbuffer setupBuffers");

        GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER,GLES20.GL_TEXTURE_2D,unityTextureID,0);
        checkGlError("glFrameBufferTexture2D");

        checkFrameBufferStatus();

        GLES20.glClearColor(1.0f,0.5f,0.0f,1.0f);
        checkGlError("glClearColor setupBuffers");
        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
        checkGlError("glClear setupBuffers");
    }

    public void DrawFrame()
    {
        if(isNewFrame && mSTMatrix != null) {

            int[] testBuffer = new int[1];
            GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING,testBuffer,0);

            Log.d("Unity","DrawFrame binded = " + testBuffer[0] + " idFBO = " + idFBO);

            GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER,idFBO);
            checkGlError("glBindFrameBuffer DrawFrame");

            GLES20.glClearColor(0.0f,0.2f,1.0f);
            checkGlError("glClearColor DrawFrame");
            GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
            checkGlError("glClear DrawFrame");

            GLES20.glUseProgram(glProgram);
            checkGlError("glUseProgram DrawFrame");

            GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
            checkGlError("glActiveTexture DrawFrame");
            GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,mTextureId);
            checkGlError("glBindTexture DrawFrame");

            mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
            GLES20.glVertexAttribPointer(maTextureHandle,2,GLES20.GL_FLOAT,TRIANGLE_VERTICES_DATA_STRIDE_BYTES,mTriangleVertices);
            checkGlError("glVertexAttribPointer DrawFrame");
            GLES20.glEnableVertexAttribArray(maTextureHandle);
            checkGlError("glEnableVertexAttribArray DrawFrame");

            Matrix.setIdentityM(mMVPMatrix,0);
            GLES20.glUniformMatrix4fv(muMVPMatrixHandle,1,mMVPMatrix,0);
            checkGlError("glUniformMatrix4fv MVP onFrameAvailable");
            GLES20.glUniformMatrix4fv(muSTMatrixHandle,mSTMatrix,0);
            checkGlError("glUniformMatrix4fv ST onFrameAvailable");

            GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP,4);
            checkGlError("glDrawArrays onFrameAvailable");

            GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER,0);
            checkGlError("glBindFrameBuffer 0 onFrameAvailable");
            GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,0);
            checkGlError("glBindTexture onFrameAvailable");

            isNewFrame = false;
        }
    }

    public void LaunchStream(String streamLink) {

        final String path = streamLink; //"http://dlqncdn.miaopai.com/stream/MVaux41A4lkuWloBbGUGaQ__.mp4"; //"rtmp://live.hkstv.hk.lxdns.com/live/hks";
        Log.i("Unity","hop hop1 = " + path);

        _currActivity.runOnUiThread(new Runnable() {
            @Override
            public void run() {

                _streamConnection.setVideoPath(path);
                _streamConnection.setMediaController(new MediaController(_currActivity));
                _streamConnection.requestFocus();

                _streamConnection.setOnErrorListener(new MediaPlayer.OnErrorListener() {
                    @Override
                    public boolean onError(MediaPlayer mp,int what,int extra) {
                        Log.i("Unity","some error,I don't know. what = " + what + " extra = " + extra);
                        return false;
                    }
                });

                _streamConnection.setOnPreparedListener(new MediaPlayer.OnPreparedListener() {

                    @Override
                    public void onPrepared(MediaPlayer mediaPlayer) {
                        // optional need Vitamio 4.0
                        Log.i("Unity","hop hop5");
                        mediaPlayer.setPlaybackSpeed(1.0f);

                    }
                });

                initGLTexture();

                _cachedSurfaceTexture = new SurfaceTexture(mTextureId);
                _cachedSurfaceTexture.setDefaultBufferSize(texWidth,texHeight);

                _cachedSurfaceTexture.setOnFrameAvailableListener(new SurfaceTexture.OnFrameAvailableListener() {
                    @Override
                    public void onFrameAvailable(SurfaceTexture surfaceTexture) {
                        synchronized (this) {
                            surfaceTexture.updateTexImage();

                            mSTMatrix = new float[16];
                            surfaceTexture.getTransformMatrix(mSTMatrix);
                            isNewFrame = true;
                        }
                    }
                });

                _cachedSurface = new Surface(_cachedSurfaceTexture);
                _streamConnection.setSurfaceToPlayer(_cachedSurface);

                Log.i("Unity","You're the best around!");
            }
        });
    }
}

我决定提供我的Android插件的所有代码,以便最清楚地了解我所拥有的情况.基本上,我正在尝试做的事情：
1)我从Unity端调用方法“GetTexturePtr”,它创建GL_TEXTURE_2D纹理,我将其应用于Unity Texture2D.同样在Android端我设置框架和渲染缓冲区来改变这个纹理的颜色.它工作正常,因为它完美地填充了颜色.
2)然后我调用方法“LaunchStream”,在其中创建GL_TEXTURE_EXTERNAL_OES纹理(在“initGLTexture()”方法中),此纹理适用于SurfaceTexture.
3)同样在Unity Update()方法中我调用android方法“DrawFrame()”,它应该根据SurfaceTexture更改来更新我的Unity纹理.

现在我在GLES20.glBindTexture上有glError 1282(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,mTextureId);当然,纹理只是填充绿色

GLES20.glClearColor(0.0f,1.0f);
            checkGlError("glClearColor DrawFrame");

我做错了什么？