Booth multiplier

What problems are you having? why not let us help you rather than expecting other people do your work for you?

library IEEE;
use IEEE.std_logic_1164.all;
use IEEE.std_logic_unsigned.all;
entity RECODER is
    port (
    DIN  : in  std_logic_vector(15 downto 0);
    BIT3 : in  std_logic_vector( 2 downto 0);
    DOUT : out std_logic_vector(16 downto 0));
end RECODER;
architecture RTL of RECODER is
    constant N : integer := 16;
    subtype bitn1 is std_logic_vector(N downto 0);
    
    function COMP2 (D : in bitn1) return bitn1 is
        variable Dn : bitn1; 
        begin Dn := not D;
        return(Dn+1);
        end COMP2;
        
        begin
            process (DIN, BIT3)
                begin 
                    case BIT3 is    
                        when "001" | "010" =>
                            DOUT <= DIN(N-1) & DIN;
                        when "101" | "110" =>
                            DOUT <= COMP2(DIN(N-1) & DIN);
                        when "100" =>
                            DOUT <= COMP2(DIN & '0');
                        when "011" =>
                            DOUT <= DIN & '0';
                        when others =>
                            DOUT <= (DOUT'range => '0');
                        end case;
                    end process;
                end RTL;

And whats the problem?

I have no idea if the code is correct. You need to write a testbench to test the code, and see if it works how you expect.